From pypy.commits at gmail.com Wed Jun 1 07:35:13 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 04:35:13 -0700 (PDT) Subject: [pypy-commit] pypy default: Another pydoc hack, needed because FFIWrapperFunction is otherwise not Message-ID: <574ec871.875a1c0a.fa44.ffffcda3@mx.google.com> Author: Armin Rigo Branch: Changeset: r84855:5039c4482729 Date: 2016-06-01 13:02 +0200 http://bitbucket.org/pypy/pypy/changeset/5039c4482729/ Log: Another pydoc hack, needed because FFIWrapperFunction is otherwise not considered like a function diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1784,3 +1784,9 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + + def test_FFIFunctionWrapper(self): + ffi, lib = self.prepare("void f(void);", "test_FFIFunctionWrapper", + "void f(void) { }") + assert lib.f.__get__(42) is lib.f + assert lib.f.__get__(42, int) is lib.f diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py --- a/pypy/module/_cffi_backend/wrapper.py +++ b/pypy/module/_cffi_backend/wrapper.py @@ -100,6 +100,11 @@ doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename) return space.wrap(doc) + def descr_get(self, space, w_obj, w_type=None): + # never bind anything, but a __get__ is still present so that + # pydoc displays useful information (namely, the __repr__) + return self + @jit.unroll_safe def prepare_args(space, rawfunctype, args_w, start_index): @@ -136,5 +141,6 @@ __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper), __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper), __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc), + __get__ = interp2app(W_FunctionWrapper.descr_get), ) W_FunctionWrapper.typedef.acceptable_as_base_class = False From pypy.commits at gmail.com Wed Jun 1 07:35:15 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 04:35:15 -0700 (PDT) Subject: [pypy-commit] pypy default: 'gc_bit': trying out a way that seems to generate better code with gcc. Message-ID: <574ec873.0c9c1c0a.28a9d.ffffdd9b@mx.google.com> Author: Armin Rigo Branch: Changeset: r84856:45e1ec5bcb37 Date: 2016-06-01 13:35 +0200 http://bitbucket.org/pypy/pypy/changeset/45e1ec5bcb37/ Log: 'gc_bit': trying out a way that seems to generate better code with gcc. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1357,11 +1357,14 @@ return cls.minimal_size_in_nursery def write_barrier(self, addr_struct): - if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS: + # see OP_GC_BIT in translator/c/gc.py + if llop.gc_bit(lltype.Signed, self.header(addr_struct), + GCFLAG_TRACK_YOUNG_PTRS): self.remember_young_pointer(addr_struct) def write_barrier_from_array(self, addr_array, index): - if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS: + if llop.gc_bit(lltype.Signed, self.header(addr_array), + GCFLAG_TRACK_YOUNG_PTRS): if self.card_page_indices > 0: self.remember_young_pointer_from_array2(addr_array, index) else: diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -470,6 +470,7 @@ 'gc_pin' : LLOp(canrun=True), 'gc_unpin' : LLOp(canrun=True), 'gc__is_pinned' : LLOp(canrun=True), + 'gc_bit' : LLOp(sideeffects=False, canrun=True), 'gc_get_rpy_roots' : LLOp(), 'gc_get_rpy_referents': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -1,3 +1,4 @@ +import random, sys from rpython.flowspace.operation import op from rpython.rlib import debug from rpython.rlib.rarithmetic import is_valid_int @@ -680,6 +681,11 @@ def op_gc_writebarrier(addr): pass +def op_gc_bit(hdr, bitmask): + if hdr.tid & bitmask: + return random.randrange(1, sys.maxint) + return 0 + def op_shrink_array(array, smallersize): return False diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -391,6 +391,34 @@ raise AssertionError(subopnum) return ' '.join(parts) + def OP_GC_BIT(self, funcgen, op): + # This is a two-arguments operation (x, y) where x is a + # pointer and y is a constant power of two. It returns 0 if + # "(*(Signed*)x) & y == 0", and non-zero if it is "== y". + # + # On x86-64, emitting this is better than emitting a load + # followed by an INT_AND for the case where y doesn't fit in + # 32 bits. I've seen situations where a register was wasted + # to contain the constant 2**32 throughout a complete messy + # function; the goal of this GC_BIT is to avoid that. + # + # Don't abuse, though. If you need to check several bits in + # sequence, then it's likely better to load the whole Signed + # first; using GC_BIT would result in multiple accesses to + # memory. + # + bitmask = op.args[1].value + assert bitmask > 0 and (bitmask & (bitmask - 1)) == 0 + offset = 0 + while bitmask >= 0x100: + offset += 1 + bitmask >>= 8 + if sys.byteorder == 'big': + offset = 'sizeof(Signed)-%s' % (offset+1) + return '%s = ((char *)%s)[%s] & %d;' % (funcgen.expr(op.result), + funcgen.expr(op.args[0]), + offset, bitmask) + class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy): def gettransformer(self, translator): From pypy.commits at gmail.com Wed Jun 1 10:34:53 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 07:34:53 -0700 (PDT) Subject: [pypy-commit] pypy default: Support 'x + y' if 'x' is an r_*int and 'y' is a float Message-ID: <574ef28d.22acc20a.b95a3.ffffcf12@mx.google.com> Author: Armin Rigo Branch: Changeset: r84857:9f2746f3765c Date: 2016-06-01 16:35 +0200 http://bitbucket.org/pypy/pypy/changeset/9f2746f3765c/ Log: Support 'x + y' if 'x' is an r_*int and 'y' is a float (works translated, but crashed untranslated) diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -213,6 +213,8 @@ return self_type if self_type in (bool, int, long): return other_type + if self_type is float or other_type is float: + return float if self_type.SIGNED == other_type.SIGNED: return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS)) raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type)) @@ -297,6 +299,7 @@ def _widen(self, other, value): """ if one argument is int or long, the other type wins. + if one argument is float, the result is float. otherwise, produce the largest class to hold the result. """ self_type = type(self) diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -18,11 +18,11 @@ class Test_r_int: def test__add__(self): - self.binary_test(lambda x, y: x + y) + self.binary_test(lambda x, y: x + y, includes_floats=True) def test__sub__(self): - self.binary_test(lambda x, y: x - y) + self.binary_test(lambda x, y: x - y, includes_floats=True) def test__mul__(self): - self.binary_test(lambda x, y: x * y) + self.binary_test(lambda x, y: x * y, includes_floats=True) x = 3; y = [2] assert x*y == r_int(x)*y assert y*x == y*r_int(x) @@ -58,12 +58,15 @@ cmp = f(r_int(arg)) assert res == cmp - def binary_test(self, f, rargs = None): + def binary_test(self, f, rargs=None, includes_floats=False): if not rargs: rargs = (-10, -1, 3, 55) + types_list = [(int, r_int), (r_int, int), (r_int, r_int)] + if includes_floats: + types_list += [(float, r_int), (r_int, float)] for larg in (-10, -1, 0, 3, 1234): for rarg in rargs: - for types in ((int, r_int), (r_int, int), (r_int, r_int)): + for types in types_list: res = f(larg, rarg) left, right = types cmp = f(left(larg), right(rarg)) From pypy.commits at gmail.com Wed Jun 1 11:13:30 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:13:30 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: avoid an unintended change left to lib-python Message-ID: <574efb9a.10301c0a.15400.251b@mx.google.com> Author: Armin Rigo Branch: cpyext-pickle Changeset: r84858:f9cce7de03dc Date: 2016-06-01 17:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f9cce7de03dc/ Log: avoid an unintended change left to lib-python diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py --- a/lib-python/2.7/pickle.py +++ b/lib-python/2.7/pickle.py @@ -285,6 +285,7 @@ if f: f(self, obj) # Call unbound method with explicit self return + # Check copy_reg.dispatch_table reduce = dispatch_table.get(t) if reduce: From pypy.commits at gmail.com Wed Jun 1 11:16:01 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 08:16:01 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: use space.appexec (arigato) Message-ID: <574efc31.cc1d1c0a.3c4ee.4fc6@mx.google.com> Author: Matti Picus Branch: cpyext-pickle Changeset: r84859:5d72cee32acb Date: 2016-06-01 18:14 +0300 http://bitbucket.org/pypy/pypy/changeset/5d72cee32acb/ Log: use space.appexec (arigato) diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -3,12 +3,6 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import api -add_pickle_key = gateway.applevel(''' - def add_pickle_key(methodtype): - from pickle import Pickler - Pickler.dispatch[methodtype] = Pickler.save_global -''', filename=__file__).interphook('add_pickle_key') - class Module(MixedModule): interpleveldefs = { 'load_module': 'api.load_extension_module', @@ -23,7 +17,10 @@ space.fromcache(State).startup(space) method = pypy.module.cpyext.typeobject.get_new_method_def(space) w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) - add_pickle_key(space, space.type(w_obj)) + space.appexec([space.type(w_obj)], """(methodtype): + from pickle import Pickler + Pickler.dispatch[methodtype] = Pickler.save_global + """) def register_atexit(self, function): if len(self.atexit_funcs) >= 32: From pypy.commits at gmail.com Wed Jun 1 11:30:07 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 08:30:07 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: more simplification (arigato) Message-ID: <574eff7f.2946c20a.daddb.ffffbac9@mx.google.com> Author: Matti Picus Branch: cpyext-pickle Changeset: r84860:7ce632bcc863 Date: 2016-06-01 18:29 +0300 http://bitbucket.org/pypy/pypy/changeset/7ce632bcc863/ Log: more simplification (arigato) diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -15,9 +15,8 @@ def startup(self, space): space.fromcache(State).startup(space) - method = pypy.module.cpyext.typeobject.get_new_method_def(space) - w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) - space.appexec([space.type(w_obj)], """(methodtype): + w_obj = space.gettypefor(pypy.module.cpyext.methodobject.W_PyCFunctionObject) + space.appexec([w_obj], """(methodtype): from pickle import Pickler Pickler.dispatch[methodtype] = Pickler.save_global """) From pypy.commits at gmail.com Wed Jun 1 11:32:45 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:32:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Move the CollectAnalyzer to 'backendopt' and use it from the JIT codewriter Message-ID: <574f001d.2450c20a.9faa1.fffffba3@mx.google.com> Author: Armin Rigo Branch: Changeset: r84861:211242489109 Date: 2016-06-01 15:37 +0200 http://bitbucket.org/pypy/pypy/changeset/211242489109/ Log: Move the CollectAnalyzer to 'backendopt' and use it from the JIT codewriter diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -14,6 +14,7 @@ from rpython.translator.backendopt.canraise import RaiseAnalyzer from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer from rpython.translator.backendopt.graphanalyze import DependencyTracker +from rpython.translator.backendopt.collectanalyze import CollectAnalyzer class CallControl(object): @@ -37,9 +38,9 @@ self.virtualizable_analyzer = VirtualizableAnalyzer(translator) self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator) self.randomeffects_analyzer = RandomEffectsAnalyzer(translator) - self.seen = DependencyTracker(self.readwrite_analyzer) - else: - self.seen = None + self.collect_analyzer = CollectAnalyzer(translator) + self.seen_rw = DependencyTracker(self.readwrite_analyzer) + self.seen_gc = DependencyTracker(self.collect_analyzer) # for index, jd in enumerate(jitdrivers_sd): jd.index = index @@ -294,9 +295,9 @@ "but the function has no result" % (op, )) # effectinfo = effectinfo_from_writeanalyze( - self.readwrite_analyzer.analyze(op, self.seen), self.cpu, + self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, - extradescr, + extradescr, self.collect_analyzer.analyze(op, self.seen_gc), ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -116,7 +116,8 @@ oopspecindex=OS_NONE, can_invalidate=False, call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET, - extradescrs=None): + extradescrs=None, + can_collect=True): readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields) readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays) readonly_descrs_interiorfields = frozenset_or_none( @@ -133,7 +134,8 @@ write_descrs_interiorfields, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + can_collect) tgt_func, tgt_saveerr = call_release_gil_target if tgt_func: key += (object(),) # don't care about caching in this case @@ -184,6 +186,7 @@ # result.extraeffect = extraeffect result.can_invalidate = can_invalidate + result.can_collect = can_collect result.oopspecindex = oopspecindex result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target @@ -230,6 +233,9 @@ def check_can_invalidate(self): return self.can_invalidate + def check_can_collect(self): + return self.can_collect + def check_is_elidable(self): return (self.extraeffect == self.EF_ELIDABLE_CAN_RAISE or self.extraeffect == self.EF_ELIDABLE_OR_MEMORYERROR or @@ -268,7 +274,8 @@ can_invalidate=False, call_release_gil_target= EffectInfo._NO_CALL_RELEASE_GIL_TARGET, - extradescr=None): + extradescr=None, + can_collect=True): from rpython.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -343,6 +350,9 @@ else: assert 0 # + if extraeffect >= EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE: + can_collect = True + # return EffectInfo(readonly_descrs_fields, readonly_descrs_arrays, readonly_descrs_interiorfields, @@ -353,7 +363,8 @@ oopspecindex, can_invalidate, call_release_gil_target, - extradescr) + extradescr, + can_collect) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -334,3 +334,37 @@ assert call_op.opname == 'direct_call' with py.test.raises(Exception): call_descr = cc.getcalldescr(call_op) + +def test_can_or_cannot_collect(): + from rpython.jit.backend.llgraph.runner import LLGraphCPU + prebuilts = [[5], [6]] + l = [] + def f1(n): + if n > 1: + raise IndexError + return prebuilts[n] # cannot collect + f1._dont_inline_ = True + + def f2(n): + return [n] # can collect + f2._dont_inline_ = True + + def f(n): + a = f1(n) + b = f2(n) + return len(a) + len(b) + + rtyper = support.annotate(f, [1]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + [f_graph] = [x for x in res if x.func is f] + for index, expected in [ + (0, False), # f1() + (1, True), # f2() + (2, False), # len() + (3, False)]: # len() + call_op = f_graph.startblock.operations[index] + assert call_op.opname == 'direct_call' + call_descr = cc.getcalldescr(call_op) + assert call_descr.extrainfo.check_can_collect() == expected diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -5,7 +5,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper import rmodel, annlowlevel from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup -from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS, llop +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.memory import gctypelayout from rpython.memory.gctransform.log import log from rpython.memory.gctransform.support import get_rtti, ll_call_destructor @@ -14,7 +14,7 @@ from rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF, WEAKREFPTR from rpython.memory.gctypelayout import FIN_TRIGGER_FUNC, FIN_HANDLER_ARRAY from rpython.tool.sourcetools import func_with_new_name -from rpython.translator.backendopt import graphanalyze +from rpython.translator.backendopt.collectanalyze import CollectAnalyzer from rpython.translator.backendopt.finalizer import FinalizerAnalyzer from rpython.translator.backendopt.support import var_needsgc import types @@ -23,33 +23,6 @@ TYPE_ID = llgroup.HALFWORD -class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): - - def analyze_direct_call(self, graph, seen=None): - try: - func = graph.func - except AttributeError: - pass - else: - if getattr(func, '_gctransformer_hint_cannot_collect_', False): - return False - if getattr(func, '_gctransformer_hint_close_stack_', False): - return True - return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, - seen) - def analyze_external_call(self, funcobj, seen=None): - if funcobj.random_effects_on_gcobjs: - return True - return graphanalyze.BoolGraphAnalyzer.analyze_external_call( - self, funcobj, seen) - def analyze_simple_operation(self, op, graphinfo): - if op.opname in ('malloc', 'malloc_varsize'): - flags = op.args[1].value - return flags['flavor'] == 'gc' - else: - return (op.opname in LL_OPERATIONS and - LL_OPERATIONS[op.opname].canmallocgc) - def propagate_no_write_barrier_needed(result, block, mallocvars, collect_analyzer, entrymap, startindex=0): diff --git a/rpython/translator/backendopt/collectanalyze.py b/rpython/translator/backendopt/collectanalyze.py new file mode 100644 --- /dev/null +++ b/rpython/translator/backendopt/collectanalyze.py @@ -0,0 +1,33 @@ +from rpython.translator.backendopt import graphanalyze +from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS + +# NB. tests are in rpython/memory/gctransform/test/test_framework.py + + +class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): + + def analyze_direct_call(self, graph, seen=None): + try: + func = graph.func + except AttributeError: + pass + else: + if getattr(func, '_gctransformer_hint_cannot_collect_', False): + return False + if getattr(func, '_gctransformer_hint_close_stack_', False): + return True + return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, + seen) + def analyze_external_call(self, funcobj, seen=None): + if funcobj.random_effects_on_gcobjs: + return True + return graphanalyze.BoolGraphAnalyzer.analyze_external_call( + self, funcobj, seen) + + def analyze_simple_operation(self, op, graphinfo): + if op.opname in ('malloc', 'malloc_varsize'): + flags = op.args[1].value + return flags['flavor'] == 'gc' + else: + return (op.opname in LL_OPERATIONS and + LL_OPERATIONS[op.opname].canmallocgc) From pypy.commits at gmail.com Wed Jun 1 11:32:46 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:32:46 -0700 (PDT) Subject: [pypy-commit] pypy default: Don't spill registers with pointers around calls that cannot collect Message-ID: <574f001e.41561c0a.2652a.43a2@mx.google.com> Author: Armin Rigo Branch: Changeset: r84862:6aa8ce23a942 Date: 2016-06-01 16:01 +0200 http://bitbucket.org/pypy/pypy/changeset/6aa8ce23a942/ Log: Don't spill registers with pointers around calls that cannot collect diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -397,9 +397,9 @@ else: self.rm.force_spill_var(var) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.vfprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.vfprm.before_call(save_all_regs) def _sync_var(self, v): if v.type == FLOAT: @@ -552,8 +552,7 @@ prepare_op_call_f = _prepare_op_call prepare_op_call_n = _prepare_op_call - def _prepare_call(self, op, force_store=[], save_all_regs=False, - first_arg_index=1): + def _prepare_call(self, op, save_all_regs=False, first_arg_index=1): args = [None] * (op.numargs() + 3) calldescr = op.getdescr() assert isinstance(calldescr, CallDescr) @@ -571,17 +570,27 @@ args[1] = imm(size) args[2] = sign_loc - args[0] = self._call(op, args, force_store, save_all_regs) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + + args[0] = self._call(op, args, gc_level) return args - def _call(self, op, arglocs, force_store=[], save_all_regs=False): - # spill variables that need to be saved around calls - self.vfprm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + def _call(self, op, arglocs, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 + self.vfprm.before_call(save_all_regs=save_all_regs) + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 + self.rm.before_call(save_all_regs=save_all_regs) resloc = self.after_call(op) return resloc @@ -1068,7 +1077,7 @@ def _prepare_op_call_assembler(self, op, fcond): locs = self.locs_for_call_assembler(op) tmploc = self.get_scratch_reg(INT, selected_reg=r.r0) - resloc = self._call(op, locs + [tmploc], save_all_regs=True) + resloc = self._call(op, locs + [tmploc], gc_level=2) return locs + [resloc, tmploc] prepare_op_call_assembler_i = _prepare_op_call_assembler diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -369,9 +369,9 @@ # This operation is used only for testing self.force_spill_var(op.getarg(0)) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.fprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.fprm.before_call(save_all_regs) def after_call(self, v): if v.type == FLOAT: @@ -756,7 +756,7 @@ src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(save_all_regs=False) + self._spill_before_call(gc_level=0) return [src_ptr_loc, dst_ptr_loc, src_ofs_loc, dst_ofs_loc, length_loc] @@ -789,13 +789,15 @@ prepare_call_f = _prepare_call prepare_call_n = _prepare_call - def _spill_before_call(self, save_all_regs=False): - # spill variables that need to be saved around calls + def _spill_before_call(self, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 self.fprm.before_call(save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) def _prepare_call(self, op, save_all_regs=False): @@ -803,7 +805,18 @@ args.append(None) for i in range(op.numargs()): args.append(self.loc(op.getarg(i))) - self._spill_before_call(save_all_regs) + + calldescr = op.getdescr() + assert isinstance(calldescr, CallDescr) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + self._spill_before_call(gc_level=gc_level) + if op.type != VOID: resloc = self.after_call(op) args[0] = resloc @@ -932,7 +945,7 @@ def _prepare_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._spill_before_call(save_all_regs=True) + self._spill_before_call(gc_level=2) if op.type != VOID: resloc = self.after_call(op) else: diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -795,22 +795,22 @@ else: self._consider_call(op) - def _call(self, op, arglocs, force_store=[], guard_not_forced=False): + def _call(self, op, arglocs, gc_level): # we need to save registers on the stack: # # - at least the non-callee-saved registers # - # - we assume that any call can collect, and we - # save also the callee-saved registers that contain GC pointers + # - if gc_level > 0, we save also the callee-saved registers that + # contain GC pointers # - # - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs - # anyway, in case we need to do cpu.force(). The issue is that - # grab_frame_values() would not be able to locate values in - # callee-saved registers. + # - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER. We + # have to save all regs anyway, in case we need to do + # cpu.force(). The issue is that grab_frame_values() would + # not be able to locate values in callee-saved registers. # - save_all_regs = guard_not_forced - self.xrm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: + save_all_regs = gc_level == 2 + self.xrm.before_call(save_all_regs=save_all_regs) + if gc_level == 1: gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap # we save all the registers for shadowstack and asmgcc for now # --- for asmgcc too: we can't say "register x is a gc ref" @@ -818,7 +818,7 @@ # more for now. if gcrootmap: # and gcrootmap.is_shadow_stack: save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + self.rm.before_call(save_all_regs=save_all_regs) if op.type != 'v': if op.type == FLOAT: resloc = self.xrm.after_call(op) @@ -838,9 +838,18 @@ sign_loc = imm1 else: sign_loc = imm0 + # + effectinfo = calldescr.get_extra_info() + if guard_not_forced: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + # self._call(op, [imm(size), sign_loc] + [self.loc(op.getarg(i)) for i in range(op.numargs())], - guard_not_forced=guard_not_forced) + gc_level=gc_level) def _consider_real_call(self, op): effectinfo = op.getdescr().get_extra_info() @@ -899,7 +908,7 @@ def _consider_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._call(op, locs, guard_not_forced=True) + self._call(op, locs, gc_level=2) consider_call_assembler_i = _consider_call_assembler consider_call_assembler_r = _consider_call_assembler consider_call_assembler_f = _consider_call_assembler From pypy.commits at gmail.com Wed Jun 1 11:32:48 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:32:48 -0700 (PDT) Subject: [pypy-commit] pypy default: Forgot to use the gcmap-free version here Message-ID: <574f0020.697ac20a.839d1.ffffe8e1@mx.google.com> Author: Armin Rigo Branch: Changeset: r84863:d7d7ea3c1caa Date: 2016-06-01 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/d7d7ea3c1caa/ Log: Forgot to use the gcmap-free version here diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -467,7 +467,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() return fcond def _genop_same_as(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -603,7 +603,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _genop_call(self, op, arglocs, regalloc): oopspecindex = regalloc.get_oopspecindex(op) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2112,7 +2112,11 @@ assert isinstance(saveerrloc, ImmedLoc) cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _store_force_index(self, guard_op): assert (guard_op.getopnum() == rop.GUARD_NOT_FORCED or From pypy.commits at gmail.com Wed Jun 1 11:32:50 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:32:50 -0700 (PDT) Subject: [pypy-commit] pypy default: ppc fix Message-ID: <574f0022.029a1c0a.673e9.4786@mx.google.com> Author: Armin Rigo Branch: Changeset: r84864:8f34fad80e6d Date: 2016-06-01 16:51 +0200 http://bitbucket.org/pypy/pypy/changeset/8f34fad80e6d/ Log: ppc fix diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -1,6 +1,7 @@ from rpython.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager, TempVar, compute_vars_longevity, BaseRegalloc) +from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32) from rpython.jit.codewriter import longlong from rpython.jit.backend.ppc.jump import (remap_frame_layout, From pypy.commits at gmail.com Wed Jun 1 11:32:51 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 08:32:51 -0700 (PDT) Subject: [pypy-commit] pypy default: merge heads Message-ID: <574f0023.011f1c0a.70560.1556@mx.google.com> Author: Armin Rigo Branch: Changeset: r84865:2611d008c62d Date: 2016-06-01 17:33 +0200 http://bitbucket.org/pypy/pypy/changeset/2611d008c62d/ Log: merge heads diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -213,6 +213,8 @@ return self_type if self_type in (bool, int, long): return other_type + if self_type is float or other_type is float: + return float if self_type.SIGNED == other_type.SIGNED: return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS)) raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type)) @@ -297,6 +299,7 @@ def _widen(self, other, value): """ if one argument is int or long, the other type wins. + if one argument is float, the result is float. otherwise, produce the largest class to hold the result. """ self_type = type(self) diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -18,11 +18,11 @@ class Test_r_int: def test__add__(self): - self.binary_test(lambda x, y: x + y) + self.binary_test(lambda x, y: x + y, includes_floats=True) def test__sub__(self): - self.binary_test(lambda x, y: x - y) + self.binary_test(lambda x, y: x - y, includes_floats=True) def test__mul__(self): - self.binary_test(lambda x, y: x * y) + self.binary_test(lambda x, y: x * y, includes_floats=True) x = 3; y = [2] assert x*y == r_int(x)*y assert y*x == y*r_int(x) @@ -58,12 +58,15 @@ cmp = f(r_int(arg)) assert res == cmp - def binary_test(self, f, rargs = None): + def binary_test(self, f, rargs=None, includes_floats=False): if not rargs: rargs = (-10, -1, 3, 55) + types_list = [(int, r_int), (r_int, int), (r_int, r_int)] + if includes_floats: + types_list += [(float, r_int), (r_int, float)] for larg in (-10, -1, 0, 3, 1234): for rarg in rargs: - for types in ((int, r_int), (r_int, int), (r_int, r_int)): + for types in types_list: res = f(larg, rarg) left, right = types cmp = f(left(larg), right(rarg)) From pypy.commits at gmail.com Wed Jun 1 13:18:54 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 01 Jun 2016 10:18:54 -0700 (PDT) Subject: [pypy-commit] pypy default: Improve the algorithm in before_call(): it now tries to emit some Message-ID: <574f18fe.e153c20a.20c17.4d83@mx.google.com> Author: Armin Rigo Branch: Changeset: r84866:c65cfaeb741a Date: 2016-06-01 19:19 +0200 http://bitbucket.org/pypy/pypy/changeset/c65cfaeb741a/ Log: Improve the algorithm in before_call(): it now tries to emit some register-register moves if possible, instead of spilling to the jitframe. diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -574,27 +574,113 @@ self.assembler.regalloc_mov(reg, to) # otherwise it's clean + def _bc_spill(self, v, new_free_regs): + self._sync_var(v) + new_free_regs.append(self.reg_bindings.pop(v)) + def before_call(self, force_store=[], save_all_regs=0): - """ Spill registers before a call, as described by - 'self.save_around_call_regs'. Registers are not spilled if - they don't survive past the current operation, unless they - are listed in 'force_store'. 'save_all_regs' can be 0 (default), - 1 (save all), or 2 (save default+PTRs). + """Spill or move some registers before a call. By default, + this means: for every register in 'self.save_around_call_regs', + if there is a variable there and it survives longer than + the current operation, then it is spilled/moved somewhere else. + + 'save_all_regs' can be 0 (default set of registers), 1 (do that + for all registers), or 2 (default + gc ptrs). + + Overview of what we do (the implementation does it differently, + for the same result): + + * we first check the set of registers that are free: call it F. + + * possibly_free_vars() is implied for all variables (except + the ones listed in force_store): if they don't survive past + the current operation, they are forgotten now. (Their + register remain not in F, because they are typically + arguments to the call, so they should not be overwritten by + the next step.) + + * then for every variable that needs to be spilled/moved: if + there is an entry in F that is acceptable, pick it and emit a + move. Otherwise, emit a spill. Start doing this with the + variables that survive the shortest time, to give them a + better change to remain in a register---similar algo as + _pick_variable_to_spill(). + + Note: when a register is moved, it often (but not always) means + we could have been more clever and picked a better register in + the first place, when we did so earlier. It is done this way + anyway, as a local hack in this function, because on x86 CPUs + such register-register moves are almost free. """ + new_free_regs = [] + move_or_spill = [] + for v, reg in self.reg_bindings.items(): - if v not in force_store and self.longevity[v][1] <= self.position: + max_age = self.longevity[v][1] + if v not in force_store and max_age <= self.position: # variable dies del self.reg_bindings[v] - self.free_regs.append(reg) + new_free_regs.append(reg) continue - if save_all_regs != 1 and reg not in self.save_around_call_regs: - if save_all_regs == 0: - continue # we don't have to - if v.type != REF: - continue # only save GC pointers - self._sync_var(v) - del self.reg_bindings[v] - self.free_regs.append(reg) + + if save_all_regs == 1: + # we need to spill all registers in this mode + self._bc_spill(v, new_free_regs) + # + elif save_all_regs == 2 and v.type == REF: + # we need to spill all GC ptrs in this mode + self._bc_spill(v, new_free_regs) + # + elif reg not in self.save_around_call_regs: + continue # in a register like ebx/rbx: it is fine where it is + # + else: + # this is a register like eax/rax, which needs either + # spilling or moving. + move_or_spill.append((v, max_age)) + + if len(move_or_spill) > 0: + while len(self.free_regs) > 0: + new_reg = self.free_regs.pop() + if new_reg in self.save_around_call_regs: + new_free_regs.append(new_reg) # not this register... + continue + # This 'new_reg' is suitable for moving a candidate to. + # Pick the one with the smallest max_age. (This + # is one step of a naive sorting algo, slow in theory, + # but the list should always be very small so it + # doesn't matter.) + best_i = 0 + smallest_max_age = move_or_spill[0][1] + for i in range(1, len(move_or_spill)): + max_age = move_or_spill[i][1] + if max_age < smallest_max_age: + best_i = i + smallest_max_age = max_age + v, max_age = move_or_spill.pop(best_i) + # move from 'reg' to 'new_reg' + reg = self.reg_bindings[v] + if not we_are_translated(): + if move_or_spill: + assert max_age <= min([_a for _, _a in move_or_spill]) + assert reg in self.save_around_call_regs + assert new_reg not in self.save_around_call_regs + self.assembler.regalloc_mov(reg, new_reg) + self.reg_bindings[v] = new_reg # change the binding + new_free_regs.append(reg) + # + if len(move_or_spill) == 0: + break + else: + # no more free registers to move to, spill the rest + for v, max_age in move_or_spill: + self._bc_spill(v, new_free_regs) + + # re-add registers in 'new_free_regs', but in reverse order, + # so that the last ones (added just above, from + # save_around_call_regs) are picked last by future '.pop()' + while len(new_free_regs) > 0: + self.free_regs.append(new_free_regs.pop()) def after_call(self, v): """ Adjust registers according to the result of the call, From pypy.commits at gmail.com Wed Jun 1 15:21:23 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 12:21:23 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: start to implement ByteArrayObject without exposing CAPI ByteArrayObject.ob_* attributes Message-ID: <574f35b3.430ac20a.b0d22.4b31@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84867:0c496ba45fe1 Date: 2016-06-01 22:10 +0300 http://bitbucket.org/pypy/pypy/changeset/0c496ba45fe1/ Log: start to implement ByteArrayObject without exposing CAPI ByteArrayObject.ob_* attributes diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rtyper.lltypesystem import rffi, lltype +from pypy.objspace.std.bytearrayobject import new_bytearray from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, build_type_checkers, PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) @@ -25,18 +26,18 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) -PyByteArrayObjectFields = PyVarObjectFields + \ - (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +PyByteArrayObjectFields = PyVarObjectFields +# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) @bootstrap_function def init_bytearrayobject(space): "Type description of PyByteArrayObject" - make_typedescr(space.w_str.layout.typedef, - basestruct=PyByteArrayObject.TO, - attach=bytearray_attach, - dealloc=bytearray_dealloc, - realize=bytearray_realize) + #make_typedescr(space.w_bytearray.layout.typedef, + # basestruct=PyByteArrayObject.TO, + # attach=bytearray_attach, + # dealloc=bytearray_dealloc, + # realize=bytearray_realize) PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") @@ -83,11 +84,16 @@ XXX expand about the buffer protocol, at least somewhere""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject) -def PyByteArray_FromStringAndSize(space, string, len): + at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True) +def PyByteArray_FromStringAndSize(space, char_p, length): """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" - raise NotImplementedError + if char_p: + s = rffi.charpsize2str(char_p, length) + else: + s = length + w_buffer = space.call_function(space.w_bytearray, space.wrap(s)) + return make_ref(space, w_buffer) @cpython_api([PyObject, PyObject], PyObject) def PyByteArray_Concat(space, a, b): @@ -95,12 +101,14 @@ raise NotImplementedError @cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_Size(space, bytearray): +def PyByteArray_Size(space, w_obj): """Return the size of bytearray after checking for a NULL pointer.""" - raise NotImplementedError + if not w_obj: + return 0 + return space.len_w(w_obj) - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AsString(space, bytearray): + at cpython_api([PyObject], rffi.CCHARP, error=0) +def PyByteArray_AsString(space, w_obj): """Return the contents of bytearray as a char array after checking for a NULL pointer.""" raise NotImplementedError diff --git a/pypy/module/cpyext/include/bytearrayobject.h b/pypy/module/cpyext/include/bytearrayobject.h --- a/pypy/module/cpyext/include/bytearrayobject.h +++ b/pypy/module/cpyext/include/bytearrayobject.h @@ -14,17 +14,17 @@ * Bytes are not characters; they may be used to encode characters. * The only way to go between bytes and str/unicode is via encoding * and decoding. - * For the convenience of C programmers, the bytes type is considered - * to contain a char pointer, not an unsigned char pointer. + * While CPython exposes interfaces to this object, pypy does not */ /* Object layout */ typedef struct { PyObject_VAR_HEAD - /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ +#if 0 int ob_exports; /* how many buffer exports */ Py_ssize_t ob_alloc; /* How many bytes allocated */ char *ob_bytes; +#endif } PyByteArrayObject; #ifdef __cplusplus diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -22,7 +22,7 @@ result = 1; } #ifdef PYPY_VERSION - expected_size = sizeof(void*)*7; + expected_size = sizeof(void*)*3; #elif defined Py_DEBUG expected_size = 64; #else @@ -62,6 +62,8 @@ return NULL; Py_DECREF(t); c = PyByteArray_AsString(s); + if (c == NULL) + return NULL; c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -78,7 +80,7 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - PyByteArrayObject *obj; + PyObject *obj; char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) @@ -118,6 +120,22 @@ PyTuple_GetItem(args, 0)), 4); ''' ), + ("str_from_bytearray", "METH_VARARGS", + ''' + char * buf; + int n; + PyObject * obj; + obj = PyTuple_GetItem(args, 0); + buf = PyByteArray_AsString(obj); + if (buf == NULL) + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } + n = PyByteArray_Size(obj); + return PyString_FromStringAndSize(buf, n); + ''' + ), ("concat", "METH_VARARGS", """ PyObject * ret, *right, *left; @@ -136,6 +154,8 @@ return ret; """)]) assert module.bytearray_from_string("huheduwe") == "huhe" + assert module.str_from_bytearray(bytearray('abc')) == 'abc' + raises(ValueError, module.str_from_bytearray, 4.0) ret = module.concat('abc', 'def') assert ret == 'abcdef' assert not isinstance(ret, str) From pypy.commits at gmail.com Wed Jun 1 15:21:25 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 12:21:25 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: first implementation of PyByteArray_AsString, returns NULL for now Message-ID: <574f35b5.6150c20a.5ee93.ffffa236@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84868:54ced1ce3f6e Date: 2016-06-01 22:11 +0300 http://bitbucket.org/pypy/pypy/changeset/54ced1ce3f6e/ Log: first implementation of PyByteArray_AsString, returns NULL for now diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -111,7 +111,11 @@ def PyByteArray_AsString(space, w_obj): """Return the contents of bytearray as a char array after checking for a NULL pointer.""" - raise NotImplementedError + if space.isinstance_w(w_obj, space.w_bytearray): + return w_obj.nonmovable_carray(space) + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) @cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) def PyByteArray_Resize(space, bytearray, len): diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -5,6 +5,7 @@ from rpython.rlib.buffer import Buffer from rpython.rlib.rstring import StringBuilder, ByteListBuilder from rpython.rlib.debug import check_list_of_chars +from rpython.rtyper.lltypesystem import rffi from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -42,6 +43,9 @@ def charbuf_w(self, space): return ''.join(self.data) + def nonmovable_carray(self, space): + return rffi.cast(rffi.CCHARP, 0) + def _new(self, value): if value is self.data: value = value[:] From pypy.commits at gmail.com Wed Jun 1 15:21:27 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 12:21:27 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: close branch to be merged Message-ID: <574f35b7.430ac20a.b0d22.4b3a@mx.google.com> Author: Matti Picus Branch: cpyext-pickle Changeset: r84869:035336d2ff24 Date: 2016-06-01 22:12 +0300 http://bitbucket.org/pypy/pypy/changeset/035336d2ff24/ Log: close branch to be merged From pypy.commits at gmail.com Wed Jun 1 15:21:28 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 12:21:28 -0700 (PDT) Subject: [pypy-commit] pypy default: merge cpyext-pickle which allows pickling of W_PyCFunctionObject Message-ID: <574f35b8.41561c0a.2652a.ffff9a36@mx.google.com> Author: Matti Picus Branch: Changeset: r84870:c4ece9807805 Date: 2016-06-01 22:13 +0300 http://bitbucket.org/pypy/pypy/changeset/c4ece9807805/ Log: merge cpyext-pickle which allows pickling of W_PyCFunctionObject diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter import gateway from pypy.module.cpyext.state import State from pypy.module.cpyext import api @@ -14,6 +15,11 @@ def startup(self, space): space.fromcache(State).startup(space) + w_obj = space.gettypefor(pypy.module.cpyext.methodobject.W_PyCFunctionObject) + space.appexec([w_obj], """(methodtype): + from pickle import Pickler + Pickler.dispatch[methodtype] = Pickler.save_global + """) def register_atexit(self, function): if len(self.atexit_funcs) >= 32: @@ -65,6 +71,7 @@ import pypy.module.cpyext.pyfile import pypy.module.cpyext.pystrtod import pypy.module.cpyext.pytraceback +import pypy.module.cpyext.methodobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -44,8 +44,8 @@ dealloc=cfunction_dealloc) def cfunction_attach(space, py_obj, w_obj): + assert isinstance(w_obj, W_PyCFunctionObject) py_func = rffi.cast(PyCFunctionObject, py_obj) - assert isinstance(w_obj, W_PyCFunctionObject) py_func.c_m_ml = w_obj.ml py_func.c_m_self = make_ref(space, w_obj.w_self) py_func.c_m_module = make_ref(space, w_obj.w_module) diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1502,7 +1502,7 @@ static PyObject * array_reduce(arrayobject *array) { - PyObject *dict, *result, *list; + PyObject *dict, *result, *list, *mod, *obj; dict = PyObject_GetAttrString((PyObject *)array, "__dict__"); if (dict == NULL) { @@ -1512,6 +1512,18 @@ dict = Py_None; Py_INCREF(dict); } + /* Return a tuple of (callable object, typecode, values, state) */ + mod = PyImport_ImportModule("array"); + if (mod == NULL) { + Py_DECREF(dict); + return NULL; + } + obj = PyObject_GetAttrString(mod, "_reconstruct"); + Py_DECREF(mod); + if (obj == NULL) { + Py_DECREF(dict); + return NULL; + } /* Unlike in Python 3.x, we never use the more efficient memory * representation of an array for pickling. This is unfortunately * necessary to allow array objects to be unpickled by Python 3.x, @@ -1524,7 +1536,7 @@ return NULL; } result = Py_BuildValue( - "O(cO)O", Py_TYPE(array), array->ob_descr->typecode, list, dict); + "O(cO)O", obj, array->ob_descr->typecode, list, dict); Py_DECREF(list); Py_DECREF(dict); return result; @@ -1916,6 +1928,11 @@ char c; PyObject *initial = NULL, *it = NULL; struct arraydescr *descr; + if (type == NULL) + { + /* when called from _reconstruct */ + type = &Arraytype; + } if (type == &Arraytype && !_PyArg_NoKeywords("array.array()", kwds)) return NULL; @@ -2017,6 +2034,11 @@ return NULL; } +static PyObject * +_reconstruct(PyTypeObject *type, PyObject *args) +{ + return array_new(type, args, NULL); +} PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ @@ -2223,6 +2245,7 @@ /* No functions in array module. */ static PyMethodDef a_methods[] = { + {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; @@ -2244,6 +2267,8 @@ return; Py_INCREF((PyObject *)&Arraytype); + if (PyType_Ready(&Arraytype) < 0) + return; PyModule_AddObject(m, "ArrayType", (PyObject *)&Arraytype); Py_INCREF((PyObject *)&Arraytype); PyModule_AddObject(m, "array", (PyObject *)&Arraytype); diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -67,3 +67,13 @@ '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0') + + def test_pickle(self): + import pickle + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + s = pickle.dumps(arr) + # pypy exports __dict__ on cpyext objects, so the pickle picks up the {} state value + #assert s == "carray\n_reconstruct\np0\n(S'i'\np1\n(lp2\nI1\naI2\naI3\naI4\natp3\nRp4\n." + rra = pickle.loads(s) # rra is arr backwards + #assert arr.tolist() == rra.tolist() diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py --- a/pypy/objspace/std/objectobject.py +++ b/pypy/objspace/std/objectobject.py @@ -180,7 +180,13 @@ if w_reduce is not None: w_cls = space.getattr(w_obj, space.wrap('__class__')) w_cls_reduce_meth = space.getattr(w_cls, w_st_reduce) - w_cls_reduce = space.getattr(w_cls_reduce_meth, space.wrap('im_func')) + try: + w_cls_reduce = space.getattr(w_cls_reduce_meth, space.wrap('im_func')) + except OperationError as e: + # i.e. PyCFunction from cpyext + if not e.match(space, space.w_AttributeError): + raise + w_cls_reduce = space.w_None w_objtype = space.w_object w_obj_dict = space.getattr(w_objtype, space.wrap('__dict__')) w_obj_reduce = space.getitem(w_obj_dict, w_st_reduce) From pypy.commits at gmail.com Wed Jun 1 15:21:30 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 12:21:30 -0700 (PDT) Subject: [pypy-commit] pypy default: document merge branch Message-ID: <574f35ba.6513c20a.e2a8d.34c6@mx.google.com> Author: Matti Picus Branch: Changeset: r84871:fe2da372ab19 Date: 2016-06-01 22:17 +0300 http://bitbucket.org/pypy/pypy/changeset/fe2da372ab19/ Log: document merge branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -125,4 +125,9 @@ .. branch: traceviewer-common-merge-point-formats -Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. \ No newline at end of file +Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. + +.. branch: cpyext-pickle + +Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch +at cpyext import time From pypy.commits at gmail.com Wed Jun 1 16:21:01 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 01 Jun 2016 13:21:01 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Finish handling positional and keyword args on function call Message-ID: <574f43ad.03c31c0a.b7fe7.ffffb988@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84872:ae9d71da76a0 Date: 2016-06-01 22:20 +0200 http://bitbucket.org/pypy/pypy/changeset/ae9d71da76a0/ Log: Finish handling positional and keyword args on function call diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1115,8 +1115,7 @@ for elt in args: if isinstance(elt.kind, ast.Starred): # A star-arg. If we've seen positional arguments, - # pack the positional arguments into a - # tuple. + # pack the positional arguments into a tuple. if nseen != 0: ops.BUILD_TUPLE(nseen) nseen = 0 @@ -1137,18 +1136,48 @@ # Pack up any trailing positional arguments. ops.BUILD_TUPLE(nseen) nsubargs += 1 - #TODO - #------------old - self.visit_sequence(args) - if keywords: - self.visit_sequence(keywords) - arg |= len(keywords) << 8 - if starargs: - starargs.walkabout(self) + if nsubargs != 0: call_type |= 1 - if kwargs: - kwargs.walkabout(self) + if nsubargs > 1: + # If we ended up with more than one stararg, we need + # to concatenate them into a single sequence. + ops.BUILD_LIST_UNPACK(nsubargs) + + # Repeat procedure for keyword args + nseen = 0 # the number of keyword arguments on the stack following + for kw in keywords: + if kw.arg is None: + # A keyword argument unpacking. + if nseen != 0: + ops.BUILD_MAP(nseen) + nseen = 0 + nsubkwargs += 1 + self.visit(kw.value) # probably wrong, elt->v.Starred.value + nsubkwargs += 1 + elif nsubkwargs != 0: + # A keyword argument and we already have a dict. + ops.LOAD_CONST(kw.arg, consts) + self.visit(kw.value) + nseen += 1 + else: + # keyword argument + self.visit(kw) + nkw += 1 + if nseen != 0: + # Pack up any trailing keyword arguments. + ops.BUILD_MAP(nseen) + nsubkwargs += 1 + if nsubargs != 0: call_type |= 2 + if nsubkwargs > 1: + # Pack it all up + function_pos = n + (code & 1) + nkw + 1 + ops.BUILD_MAP_UNPACK_WITH_CALL(nsubkwargs | (function_pos << 8)) + + assert n < 1<<8 + assert nkw < 1<<24 + n |= nkw << 8; + op = 0 if call_type == 0: op = ops.CALL_FUNCTION @@ -1159,6 +1188,7 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + #TODO emip_op_arg on each call def visit_Call(self, call): self.update_position(call.lineno) From pypy.commits at gmail.com Thu Jun 2 00:40:56 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 21:40:56 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: fill tp_as_buffer slots, starts to fix PyArg_ParseTuple(.. 's*' ..) with bytearray arg Message-ID: <574fb8d8.82e01c0a.a65b8.48d7@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84873:46d519f7ebaa Date: 2016-06-02 07:38 +0300 http://bitbucket.org/pypy/pypy/changeset/46d519f7ebaa/ Log: fill tp_as_buffer slots, starts to fix PyArg_ParseTuple(.. 's*' ..) with bytearray arg diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -23,6 +23,7 @@ else: bases = [__base] self.bases = bases + # Used in cpyext to fill tp_as_buffer slots assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" self.buffer = __buffer self.heaptype = False diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -44,7 +44,7 @@ return ''.join(self.data) def nonmovable_carray(self, space): - return rffi.cast(rffi.CCHARP, 0) + return BytearrayBuffer(self.data, False).get_raw_address() def _new(self, value): if value is self.data: @@ -991,7 +991,7 @@ W_BytearrayObject.typedef = TypeDef( - "bytearray", + "bytearray", None, None, "read-write", __doc__ = BytearrayDocstrings.__doc__, __new__ = interp2app(W_BytearrayObject.descr_new), __hash__ = None, @@ -1250,6 +1250,9 @@ for i in range(len(string)): self.data[start + i] = string[i] + def get_raw_address(self): + return rffi.cast(rffi.CCHARP, 0) + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): From pypy.commits at gmail.com Thu Jun 2 00:40:58 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 01 Jun 2016 21:40:58 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: fix tests Message-ID: <574fb8da.442cc20a.e3eef.ffffedb6@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84874:5b5631d70988 Date: 2016-06-02 07:39 +0300 http://bitbucket.org/pypy/pypy/changeset/5b5631d70988/ Log: fix tests diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -63,7 +63,10 @@ Py_DECREF(t); c = PyByteArray_AsString(s); if (c == NULL) - return NULL; + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -80,13 +83,11 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - PyObject *obj; char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); - obj = (PyByteArrayObject*)base; - memcpy(PyByteArray_AS_STRING(obj), "works", 6); + memcpy(PyByteArray_AS_STRING(base), "works", 6); Py_INCREF(base); return base; """), From pypy.commits at gmail.com Thu Jun 2 01:58:12 2016 From: pypy.commits at gmail.com (devin.jeanpierre) Date: Wed, 01 Jun 2016 22:58:12 -0700 (PDT) Subject: [pypy-commit] pypy gc-forkfriendly: Initialize the GC remote_flags pointer... Message-ID: <574fcaf4.41c8c20a.2b32e.ffffdd1a@mx.google.com> Author: Devin Jeanpierre Branch: gc-forkfriendly Changeset: r84875:bbf96bee2300 Date: 2016-06-01 21:46 -0700 http://bitbucket.org/pypy/pypy/changeset/bbf96bee2300/ Log: Initialize the GC remote_flags pointer... diff --git a/rpython/memory/gc/incminimark_remoteheader.py b/rpython/memory/gc/incminimark_remoteheader.py --- a/rpython/memory/gc/incminimark_remoteheader.py +++ b/rpython/memory/gc/incminimark_remoteheader.py @@ -31,11 +31,12 @@ def init_gc_object(self, adr, typeid16, flags=0): incminimark.IncrementalMiniMarkGCBase.init_gc_object(self, adr, typeid16, flags) + hdr = llmemory.cast_adr_to_ptr(adr, lltype.Ptr(self.HDR)) # This gets compiled to nonsense like (&pypy_g_header_1433.h_tid) - # at the top level (global variable initialization). Instead, we leave - # it as NULL and lazily initialize it later. - #hdr = llmemory.cast_adr_to_ptr(adr, lltype.Ptr(self.HDR)) - #hdr.remote_flags = lltype.direct_fieldptr(hdr, 'tid') + # at the top level (global variable initialization). Instead, we set + # it to NULL and lazily initialize it later. + ## hdr.remote_flags = lltype.direct_fieldptr(hdr, 'tid') + hdr.remote_flags = lltype.nullptr(SIGNEDP.TO) def make_forwardstub(self, obj, forward_to): assert (self.header(obj).remote_flags diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1760,5 +1760,5 @@ class TestIncrementalMiniMarkGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkGC): removetypeptr = True -class TestIncrementalMiniMarkGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkRemoteHeadersGC): +class TestIncrementalMiniMarkRemoteHeadersGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkRemoteHeadersGC): removetypeptr = True From pypy.commits at gmail.com Thu Jun 2 05:22:18 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 02:22:18 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: Backed out changeset 7ce632bcc863 Message-ID: <574ffaca.6a56c20a.a1835.3675@mx.google.com> Author: Armin Rigo Branch: cpyext-pickle Changeset: r84876:e69c12f4b550 Date: 2016-06-02 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/e69c12f4b550/ Log: Backed out changeset 7ce632bcc863 Unknown why, but test_ztranslation is not happy. I tried making test_ztranslation actually test all cpython_api functions---it is not, right now---but it opened another can of worms. diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -15,8 +15,9 @@ def startup(self, space): space.fromcache(State).startup(space) - w_obj = space.gettypefor(pypy.module.cpyext.methodobject.W_PyCFunctionObject) - space.appexec([w_obj], """(methodtype): + method = pypy.module.cpyext.typeobject.get_new_method_def(space) + w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) + space.appexec([space.type(w_obj)], """(methodtype): from pickle import Pickler Pickler.dispatch[methodtype] = Pickler.save_global """) From pypy.commits at gmail.com Thu Jun 2 05:22:19 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 02:22:19 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-pickle: reclose branch Message-ID: <574ffacb.81301c0a.92898.ffffeb1e@mx.google.com> Author: Armin Rigo Branch: cpyext-pickle Changeset: r84877:8758db4376a3 Date: 2016-06-02 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/8758db4376a3/ Log: reclose branch From pypy.commits at gmail.com Thu Jun 2 05:22:21 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 02:22:21 -0700 (PDT) Subject: [pypy-commit] pypy default: remerge cpyext-pickle with the problematic bit backed out Message-ID: <574ffacd.cc1d1c0a.a72f5.11ce@mx.google.com> Author: Armin Rigo Branch: Changeset: r84878:8ad62753199e Date: 2016-06-02 11:23 +0200 http://bitbucket.org/pypy/pypy/changeset/8ad62753199e/ Log: remerge cpyext-pickle with the problematic bit backed out diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -15,8 +15,9 @@ def startup(self, space): space.fromcache(State).startup(space) - w_obj = space.gettypefor(pypy.module.cpyext.methodobject.W_PyCFunctionObject) - space.appexec([w_obj], """(methodtype): + method = pypy.module.cpyext.typeobject.get_new_method_def(space) + w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) + space.appexec([space.type(w_obj)], """(methodtype): from pickle import Pickler Pickler.dispatch[methodtype] = Pickler.save_global """) From pypy.commits at gmail.com Thu Jun 2 09:05:35 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 06:05:35 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: A branch to add a way to ask "give me a raw pointer to this list's Message-ID: <57502f1f.4f961c0a.50cf9.74f1@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84879:fe931afb3150 Date: 2016-06-02 10:53 +0200 http://bitbucket.org/pypy/pypy/changeset/fe931afb3150/ Log: A branch to add a way to ask "give me a raw pointer to this list's items". Only for resizable lists of primitives. Turns the GcArray nonmovable, possibly making a copy of it first. From pypy.commits at gmail.com Thu Jun 2 09:05:37 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 06:05:37 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: The hardest bit is to give a consistent behavior before translation... Message-ID: <57502f21.43921c0a.99b29.71bb@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84880:3013b870792c Date: 2016-06-02 15:05 +0200 http://bitbucket.org/pypy/pypy/changeset/3013b870792c/ Log: The hardest bit is to give a consistent behavior before translation... diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1001,3 +1001,194 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.genop('gc_gettypeid', hop.args_v, resulttype=lltype.Signed) + +# ____________________________________________________________ + + +class _rawptr_missing_item(object): + pass +_rawptr_missing_item = _rawptr_missing_item() + + +class ListSupportingRawPtr(list): + """Calling this class is a no-op after translation. Before + translation, it returns a new instance of ListSupportingRawPtr, + on which rgc.nonmoving_raw_ptr_for_resizable_list() might be + used if needed. For now, only supports lists of chars. + """ + __slots__ = ('_raw_items',) # either None or a rffi.CArray(Char) + + def __init__(self, lst): + self._raw_items = None + self.__from_list(lst) + + def __resize(self): + """Called before an operation changes the size of the list""" + if self._raw_items is not None: + list.__init__(self, self.__as_list()) + self._raw_items = None + + def __from_list(self, lst): + """Initialize the list from a copy of the list 'lst'.""" + assert isinstance(lst, list) + for x in lst: + assert isinstance(x, str) and len(x) == 1 + if self is lst: + return + if len(self) != len(lst): + self.__resize() + if self._raw_items is None: + list.__init__(self, lst) + else: + assert len(self) == self._raw_items._obj.getlength() == len(lst) + for i in range(len(self)): + self._raw_items[i] = lst[i] + + def __as_list(self): + """Return a list (the same or a different one) which contains the + items in the regular way.""" + if self._raw_items is None: + return self + length = self._raw_items._obj.getlength() + assert length == len(self) + return [self._raw_items[i] for i in range(length)] + + def __getitem__(self, index): + if self._raw_items is None: + return list.__getitem__(self, index) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + return self._raw_items[index] + + def __setitem__(self, index, new): + if self._raw_items is None: + return list.__setitem__(self, index, new) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + self._raw_items[index] = new + + def __delitem__(self, index): + self.__resize() + list.__delitem__(self, index) + + def __getslice__(self, i, j): + return list.__getslice__(self.__as_list(), i, j) + + def __setslice__(self, i, j, new): + lst = self.__as_list() + list.__setslice__(lst, i, j, new) + self.__from_list(lst) + + def __delslice__(self, i, j): + lst = self.__as_list() + list.__delslice__(lst, i, j) + self.__from_list(lst) + + def __iter__(self): + try: + i = 0 + while True: + yield self[i] + i += 1 + except IndexError: + pass + + def __reversed__(self): + i = len(self) + while i > 0: + i -= 1 + yield self[i] + + def __contains__(self, item): + return list.__contains__(self.__as_list(), item) + + def __add__(self, other): + return list.__add__(self.__as_list(), other) + + def __radd__(self, other): + other = list(other) + return list.__add__(other, self) + + def __iadd__(self, other): + self.__resize() + return list.__iadd__(self, other) + + def __eq__(self, other): + return list.__eq__(self.__as_list(), other) + def __ne__(self, other): + return list.__ne__(self.__as_list(), other) + def __ge__(self, other): + return list.__ge__(self.__as_list(), other) + def __gt__(self, other): + return list.__gt__(self.__as_list(), other) + def __le__(self, other): + return list.__le__(self.__as_list(), other) + def __lt__(self, other): + return list.__lt__(self.__as_list(), other) + + def __mul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __rmul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __imul__(self, other): + self.__resize() + return list.__imul__(self, other) + + def __repr__(self): + return 'ListSupportingRawPtr(%s)' % (list.__repr__(self.__as_list()),) + + def append(self, object): + self.__resize() + return list.append(self, object) + + def count(self, value): + return list.count(self.__as_list(), value) + + def extend(self, iterable): + self.__resize() + return list.extend(self, iterable) + + def index(self, value, *start_stop): + return list.index(self.__as_list(), value, *start_stop) + + def insert(self, index, object): + self.__resize() + return list.insert(self, index, object) + + def pop(self, *opt_index): + self.__resize() + return list.pop(self, *opt_index) + + def remove(self, value): + self.__resize() + return list.remove(self, value) + + def reverse(self): + lst = self.__as_list() + list.reverse(lst) + self.__from_list(lst) + + def sort(self, *args, **kwds): + lst = self.__as_list() + list.sort(lst, *args, **kwds) + self.__from_list(lst) + + def _nonmoving_raw_ptr_for_resizable_list(self): + if self._raw_items is None: + existing_items = list(self) + from rpython.rtyper.lltypesystem import lltype, rffi + self._raw_items = lltype.malloc(rffi.CArray(lltype.Char), len(self), + flavor='raw', immortal=True) + self.__from_list(existing_items) + assert self._raw_items is not None + return self._raw_items + +def nonmoving_raw_ptr_for_resizable_list(lst): + assert isinstance(lst, ListSupportingRawPtr) + return lst._nonmoving_raw_ptr_for_resizable_list() diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.test_llinterp import gengraph, interpret from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib import rgc # Force registration of gc.collect import gc import py, sys @@ -254,6 +254,36 @@ assert typer.custom_trace_funcs == [(TP, trace_func)] +def test_nonmoving_raw_ptr_for_resizable_list(): + def f(n): + lst = ['a', 'b', 'c'] + lst = rgc.ListSupportingRawPtr(lst) + lst.append(chr(n)) + assert lst[3] == chr(n) + assert lst[-1] == chr(n) + # + ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst) + assert lst[:] == ['a', 'b', 'c', chr(n)] + assert lltype.typeOf(ptr) == rffi.CArrayPtr(lltype.Char) + assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)] + # + lst[-3] = 'X' + assert ptr[1] == 'X' + ptr[2] = 'Y' + assert lst[-2] == 'Y' + # + addr = rffi.cast(lltype.Signed, ptr) + ptr = rffi.cast(rffi.CArrayPtr(lltype.Char), addr) + lst[-4] = 'g' + assert ptr[0] == 'g' + ptr[3] = 'H' + assert lst[-1] == 'H' + return lst + # + # direct untranslated run + lst = f(35) + assert isinstance(lst, rgc.ListSupportingRawPtr) + # ____________________________________________________________ @@ -368,7 +398,6 @@ assert fq._triggered == 1 def test_finalizer_trigger_calls_too_much(self): - from rpython.rtyper.lltypesystem import lltype, rffi external_func = rffi.llexternal("foo", [], lltype.Void) # ^^^ with release_gil=True class X(object): From pypy.commits at gmail.com Thu Jun 2 09:49:31 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 06:49:31 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: Translation, step 1 Message-ID: <5750396b.2457c20a.d1c62.4827@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84881:cdb701579b36 Date: 2016-06-02 15:50 +0200 http://bitbucket.org/pypy/pypy/changeset/cdb701579b36/ Log: Translation, step 1 diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1010,13 +1010,15 @@ _rawptr_missing_item = _rawptr_missing_item() -class ListSupportingRawPtr(list): - """Calling this class is a no-op after translation. Before - translation, it returns a new instance of ListSupportingRawPtr, - on which rgc.nonmoving_raw_ptr_for_resizable_list() might be +class _ResizableListSupportingRawPtr(list): + """Calling this class is a no-op after translation. + + Before translation, it returns a new instance of + _ResizableListSupportingRawPtr, on which + rgc.nonmoving_raw_ptr_for_resizable_list() might be used if needed. For now, only supports lists of chars. """ - __slots__ = ('_raw_items',) # either None or a rffi.CArray(Char) + __slots__ = ('_raw_items',) # either None or a rffi.CCHARP def __init__(self, lst): self._raw_items = None @@ -1141,7 +1143,8 @@ return list.__imul__(self, other) def __repr__(self): - return 'ListSupportingRawPtr(%s)' % (list.__repr__(self.__as_list()),) + return '_ResizableListSupportingRawPtr(%s)' % ( + list.__repr__(self.__as_list()),) def append(self, object): self.__resize() @@ -1183,12 +1186,71 @@ if self._raw_items is None: existing_items = list(self) from rpython.rtyper.lltypesystem import lltype, rffi - self._raw_items = lltype.malloc(rffi.CArray(lltype.Char), len(self), + self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self), flavor='raw', immortal=True) self.__from_list(existing_items) assert self._raw_items is not None return self._raw_items +def resizable_list_supporting_raw_ptr(lst): + return _ResizableListSupportingRawPtr(lst) + def nonmoving_raw_ptr_for_resizable_list(lst): - assert isinstance(lst, ListSupportingRawPtr) + assert isinstance(lst, _ResizableListSupportingRawPtr) return lst._nonmoving_raw_ptr_for_resizable_list() + + +def _check_resizable_list_of_chars(s_list): + from rpython.annotator import model as annmodel + from rpython.rlib import debug + if annmodel.s_None.contains(s_list): + return # "None", will likely be generalized later + if not isinstance(s_list, annmodel.SomeList): + raise Exception("not a list, got %r" % (s_list,)) + if not isinstance(s_list.listdef.listitem.s_value, + (annmodel.SomeChar, annmodel.SomeImpossibleValue)): + raise debug.NotAListOfChars + s_list.listdef.resize() # must be resizable + +class Entry(ExtRegistryEntry): + _about_ = resizable_list_supporting_raw_ptr + + def compute_result_annotation(self, s_list): + _check_resizable_list_of_chars(s_list) + return s_list + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], 0) + +class Entry(ExtRegistryEntry): + _about_ = nonmoving_raw_ptr_for_resizable_list + + def compute_result_annotation(self, s_list): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.rtyper.llannotation import SomePtr + _check_resizable_list_of_chars(s_list) + return SomePtr(rffi.CCHARP) + + def specialize_call(self, hop): + v_list = hop.inputarg(hop.args_r[0], 0) + hop.exception_cannot_occur() # ignoring MemoryError + return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, + v_list) + +def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): + from rpython.rtyper.lltypesystem import lltype, rffi + array = ll_list.items + if can_move(array): + length = ll_list.length + new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length, + nonmovable=True) + i = 0 + while i < length: + new_array[i] = array[i] + i += 1 + ll_list.items = new_array + array = new_array + ptr = lltype.direct_arrayitems(array) + # ptr is a Ptr(FixedSizeArray(Char, 1)). Cast it to a rffi.CCHARP + return rffi.cast(rffi.CCHARP, ptr) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -257,14 +257,14 @@ def test_nonmoving_raw_ptr_for_resizable_list(): def f(n): lst = ['a', 'b', 'c'] - lst = rgc.ListSupportingRawPtr(lst) + lst = rgc.resizable_list_supporting_raw_ptr(lst) lst.append(chr(n)) assert lst[3] == chr(n) assert lst[-1] == chr(n) # ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst) assert lst[:] == ['a', 'b', 'c', chr(n)] - assert lltype.typeOf(ptr) == rffi.CArrayPtr(lltype.Char) + assert lltype.typeOf(ptr) == rffi.CCHARP assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)] # lst[-3] = 'X' @@ -273,7 +273,7 @@ assert lst[-2] == 'Y' # addr = rffi.cast(lltype.Signed, ptr) - ptr = rffi.cast(rffi.CArrayPtr(lltype.Char), addr) + ptr = rffi.cast(rffi.CCHARP, addr) lst[-4] = 'g' assert ptr[0] == 'g' ptr[3] = 'H' @@ -282,7 +282,10 @@ # # direct untranslated run lst = f(35) - assert isinstance(lst, rgc.ListSupportingRawPtr) + assert isinstance(lst, rgc._ResizableListSupportingRawPtr) + # + # llinterp run + interpret(f, [35]) # ____________________________________________________________ diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -2174,7 +2174,8 @@ def malloc(T, n=None, flavor='gc', immortal=False, zero=False, - track_allocation=True, add_memory_pressure=False): + track_allocation=True, add_memory_pressure=False, + nonmovable=False): assert flavor in ('gc', 'raw') if zero or immortal: initialization = 'example' @@ -2200,7 +2201,8 @@ @analyzer_for(malloc) def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, - s_track_allocation=None, s_add_memory_pressure=None): + s_track_allocation=None, s_add_memory_pressure=None, + s_nonmovable=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2218,6 +2220,7 @@ assert s_track_allocation is None or s_track_allocation.is_constant() assert (s_add_memory_pressure is None or s_add_memory_pressure.is_constant()) + assert s_nonmovable is None or s_nonmovable.is_constant() # not sure how to call malloc() for the example 'p' in the # presence of s_extraargs r = SomePtr(Ptr(s_T.const)) diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -659,6 +659,7 @@ a[3] = 30 a[4] = 40 b0 = direct_arrayitems(a) + assert typeOf(b0) == Ptr(FixedSizeArray(Signed, 1)) b1 = direct_ptradd(b0, 1) b2 = direct_ptradd(b1, 1) b3 = direct_ptradd(b0, 3) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -348,7 +348,7 @@ @typer_for(lltype.malloc) def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, - i_add_memory_pressure=None): + i_add_memory_pressure=None, i_nonmovable=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' @@ -357,8 +357,10 @@ (i_flavor, lltype.Void), (i_zero, None), (i_track_allocation, None), - (i_add_memory_pressure, None)) - (v_flavor, v_zero, v_track_allocation, v_add_memory_pressure) = kwds_v + (i_add_memory_pressure, None), + (i_nonmovable, None)) + (v_flavor, v_zero, v_track_allocation, + v_add_memory_pressure, v_nonmovable) = kwds_v flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value @@ -368,6 +370,8 @@ flags['track_allocation'] = v_track_allocation.value if i_add_memory_pressure is not None: flags['add_memory_pressure'] = v_add_memory_pressure.value + if i_nonmovable is not None: + flags['nonmovable'] = v_nonmovable vlist.append(hop.inputconst(lltype.Void, flags)) assert 1 <= hop.nb_args <= 2 From pypy.commits at gmail.com Thu Jun 2 10:04:00 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 07:04:00 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: An extra test Message-ID: <57503cd0.41c8c20a.4119c.48e9@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84883:1cb1fc0f1aa7 Date: 2016-06-02 16:04 +0200 http://bitbucket.org/pypy/pypy/changeset/1cb1fc0f1aa7/ Log: An extra test diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -274,6 +274,7 @@ # addr = rffi.cast(lltype.Signed, ptr) ptr = rffi.cast(rffi.CCHARP, addr) + rgc.collect() # should not move lst.items lst[-4] = 'g' assert ptr[0] == 'g' ptr[3] = 'H' From pypy.commits at gmail.com Thu Jun 2 10:03:58 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 07:03:58 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: Translation, step 2 Message-ID: <57503cce.07ecc20a.ae79d.4b32@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84882:78452109814c Date: 2016-06-02 16:00 +0200 http://bitbucket.org/pypy/pypy/changeset/78452109814c/ Log: Translation, step 2 diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -184,7 +184,7 @@ def can_move(self, addr): return False - def malloc_fixedsize_nonmovable(self, typeid): + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): raise MemoryError def pin(self, addr): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -718,8 +718,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -628,8 +628,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -557,9 +557,10 @@ getfn(func, [SomeAddress()], annmodel.s_None) - self.malloc_nonmovable_ptr = getfn(GCClass.malloc_fixedsize_nonmovable, - [s_gc, s_typeid16], - s_gcref) + self.malloc_nonmovable_ptr = getfn( + GCClass.malloc_fixed_or_varsize_nonmovable, + [s_gc, s_typeid16, annmodel.SomeInteger()], + s_gcref) self.register_finalizer_ptr = getfn(GCClass.register_finalizer, [s_gc, @@ -800,12 +801,16 @@ c_has_light_finalizer = rmodel.inputconst(lltype.Bool, has_light_finalizer) + is_varsize = op.opname.endswith('_varsize') or flags.get('varsize') + if flags.get('nonmovable'): - assert op.opname == 'malloc' - assert not flags.get('varsize') + if not is_varsize: + v_length = rmodel.inputconst(lltype.Signed, 0) + else: + v_length = op.args[-1] malloc_ptr = self.malloc_nonmovable_ptr - args = [self.c_const_gc, c_type_id] - elif not op.opname.endswith('_varsize') and not flags.get('varsize'): + args = [self.c_const_gc, c_type_id, v_length] + elif not is_varsize: zero = flags.get('zero', False) if (self.malloc_fast_ptr is not None and not c_has_finalizer.value and diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -286,6 +286,22 @@ # # llinterp run interpret(f, [35]) + # + # compilation with the GC transformer + import subprocess + from rpython.translator.interactive import Translation + # + def main(argv): + f(len(argv)) + print "OK!" + return 0 + # + t = Translation(main, gc="incminimark") + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') # ____________________________________________________________ From pypy.commits at gmail.com Thu Jun 2 10:30:59 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Jun 2016 07:30:59 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: A test for all the functions in _ResizableListSupportingRawPtr, Message-ID: <57504323.071d1c0a.d3dbd.26bc@mx.google.com> Author: Armin Rigo Branch: nonmovable-list Changeset: r84884:e975f9edeb36 Date: 2016-06-02 16:31 +0200 http://bitbucket.org/pypy/pypy/changeset/e975f9edeb36/ Log: A test for all the functions in _ResizableListSupportingRawPtr, found a problem, fix diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1109,11 +1109,14 @@ return list.__contains__(self.__as_list(), item) def __add__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() return list.__add__(self.__as_list(), other) def __radd__(self, other): - other = list(other) - return list.__add__(other, self) + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(other, self.__as_list()) def __iadd__(self, other): self.__resize() diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -304,6 +304,103 @@ data = subprocess.check_output([str(exename), '.', '.', '.']) assert data.strip().endswith('OK!') +def test_ListSupportingRawPtr_direct(): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + + def check_nonresizing(): + assert lst[1] == lst[-2] == 'b' + lst[1] = 'X' + assert lst[1] == 'X' + lst[-1] = 'Y' + assert lst[1:3] == ['X', 'Y'] + assert lst[-2:9] == ['X', 'Y'] + lst[1:2] = 'B' + assert lst[:] == ['a', 'B', 'Y'] + assert list(iter(lst)) == ['a', 'B', 'Y'] + assert list(reversed(lst)) == ['Y', 'B', 'a'] + assert 'B' in lst + assert 'b' not in lst + assert p[0] == 'a' + assert p[1] == 'B' + assert p[2] == 'Y' + assert lst + ['*'] == ['a', 'B', 'Y', '*'] + assert ['*'] + lst == ['*', 'a', 'B', 'Y'] + assert lst + lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + base = ['8'] + base += lst + assert base == ['8', 'a', 'B', 'Y'] + assert lst == ['a', 'B', 'Y'] + assert ['a', 'B', 'Y'] == lst + assert ['a', 'B', 'Z'] != lst + assert ['a', 'B', 'Z'] > lst + assert ['a', 'B', 'Z'] >= lst + assert lst * 2 == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert 2 * lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert lst.count('B') == 1 + assert lst.index('Y') == 2 + lst.reverse() + assert lst == ['Y', 'B', 'a'] + lst.sort() + assert lst == ['B', 'Y', 'a'] + lst.sort(reverse=True) + assert lst == ['a', 'Y', 'B'] + lst[1] = 'b' + lst[2] = 'c' + assert list(lst) == ['a', 'b', 'c'] + + p = lst + check_nonresizing() + assert lst._raw_items is None + lst._nonmoving_raw_ptr_for_resizable_list() + p = lst._raw_items + check_nonresizing() + assert lst._raw_items == p + assert p[0] == 'a' + assert p[1] == 'b' + assert p[2] == 'c' + + def do_resizing_operation(): + del lst[1] + yield ['a', 'c'] + + lst[:2] = ['X'] + yield ['X', 'c'] + + del lst[:2] + yield ['c'] + + x = lst + x += ['t'] + yield ['a', 'b', 'c', 't'] + + x = lst + x *= 3 + yield ['a', 'b', 'c'] * 3 + + lst.append('f') + yield ['a', 'b', 'c', 'f'] + + lst.extend('fg') + yield ['a', 'b', 'c', 'f', 'g'] + + lst.insert(1, 'k') + yield ['a', 'k', 'b', 'c'] + + n = lst.pop(1) + assert n == 'b' + yield ['a', 'c'] + + lst.remove('c') + yield ['a', 'b'] + + assert lst == ['a', 'b', 'c'] + for expect in do_resizing_operation(): + assert lst == expect + assert lst._raw_items is None + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst._nonmoving_raw_ptr_for_resizable_list() # ____________________________________________________________ From pypy.commits at gmail.com Thu Jun 2 12:26:02 2016 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Jun 2016 09:26:02 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <57505e1a.a60ac20a.20ce9.ffff8356@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r84885:1de2e9ff0c99 Date: 2016-06-02 17:04 +0100 http://bitbucket.org/pypy/pypy/changeset/1de2e9ff0c99/ Log: hg merge default diff too long, truncating to 2000 out of 5903 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -23,3 +23,5 @@ 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -834,54 +834,63 @@ c2pread, c2pwrite = None, None errread, errwrite = None, None + ispread = False if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = _subprocess.CreatePipe(None, 0) + ispread = True elif stdin == PIPE: p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) + ispread = True elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) + p2cread = self._make_inheritable(p2cread, ispread) # We just duplicated the handle, it has to be closed at the end to_close.add(p2cread) if stdin == PIPE: to_close.add(p2cwrite) + ispwrite = False if stdout is None: c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stdout == PIPE: c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) + c2pwrite = self._make_inheritable(c2pwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(c2pwrite) if stdout == PIPE: to_close.add(c2pread) + ispwrite = False if stderr is None: errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) if errwrite is None: _, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == PIPE: errread, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == STDOUT: - errwrite = c2pwrite.handle # pass id to not close it + errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) + errwrite = self._make_inheritable(errwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(errwrite) if stderr == PIPE: @@ -892,13 +901,14 @@ errread, errwrite), to_close - def _make_inheritable(self, handle): + def _make_inheritable(self, handle, close=False): """Return a duplicate of handle, which is inheritable""" dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), handle, _subprocess.GetCurrentProcess(), 0, 1, _subprocess.DUPLICATE_SAME_ACCESS) - # If the initial handle was obtained with CreatePipe, close it. - if not isinstance(handle, int): + # PyPy: If the initial handle was obtained with CreatePipe, + # close it. + if close: handle.Close() return dupl diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py --- a/lib_pypy/_pypy_irc_topic.py +++ b/lib_pypy/_pypy_irc_topic.py @@ -224,23 +224,9 @@ va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat """ -from string import ascii_uppercase, ascii_lowercase - def rot13(data): - """ A simple rot-13 encoder since `str.encode('rot13')` was removed from - Python as of version 3.0. It rotates both uppercase and lowercase letters individually. - """ - total = [] - for char in data: - if char in ascii_uppercase: - index = (ascii_uppercase.find(char) + 13) % 26 - total.append(ascii_uppercase[index]) - elif char in ascii_lowercase: - index = (ascii_lowercase.find(char) + 13) % 26 - total.append(ascii_lowercase[index]) - else: - total.append(char) - return "".join(total) + return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else + -13 if 'N'<=c.upper()<='Z' else 0)) for c in data) def some_topic(): import time diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py --- a/lib_pypy/pyrepl/simple_interact.py +++ b/lib_pypy/pyrepl/simple_interact.py @@ -43,11 +43,13 @@ return short return text -def run_multiline_interactive_console(mainmodule=None): +def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code import __main__ mainmodule = mainmodule or __main__ console = code.InteractiveConsole(mainmodule.__dict__, filename='') + if future_flags: + console.compile.compiler.flags |= future_flags def more_lines(unicodetext): # ooh, look at the hack: diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -70,9 +70,6 @@ bz2 libbz2 -lzma (PyPy3 only) - liblzma - pyexpat libexpat1 @@ -98,11 +95,16 @@ tk tk-dev +lzma (PyPy3 only) + liblzma + +To run untranslated tests, you need the Boehm garbage collector libgc. + On Debian, this is the command to install all build-time dependencies:: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ - tk-dev libgc-dev + tk-dev libgc-dev liblzma-dev For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -49,6 +49,13 @@ release-0.6 +CPython 3.3 compatible versions +------------------------------- + +.. toctree:: + + release-pypy3.3-v5.2-alpha1.rst + CPython 3.2 compatible versions ------------------------------- diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst @@ -0,0 +1,69 @@ +=================== +PyPy3 v5.2 alpha 1 +=================== + +We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the +first release of PyPy which targets Python 3.3 (3.3.5) compatibility. + +We would like to thank all of the people who donated_ to the `py3k proposal`_ +for supporting the work that went into this and future releases. + +You can download the PyPy3.3 v5.2 alpha 1 release here: + + http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2 + +Highlights +========== + +* Python 3.3.5 support! + + - Being an early alpha release, there are some `missing features`_ such as a + `PEP 393-like space efficient string representation`_ and `known issues`_ + including performance regressions (e.g. issue `#2305`_). The focus for this + release has been updating to 3.3 compatibility. Windows is also not yet + supported. + +* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4). + +What is PyPy? +============== + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT +compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems except Windows + (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Please try it out and let us know what you think. We welcome feedback, we know +you are using PyPy, please tell us about it! + +We'd especially like to thank these people for their contributions to this +release: + +Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey, +Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others. + +Cheers + +The PyPy Team + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation +.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement +.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29 +.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305 +.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip +.. _`dynamic languages`: http://pypyjs.org diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,29 @@ Fix some warnings when compiling CPython C extension modules .. branch: syntax_fix + +.. branch: remove-raisingops + +Remove most of the _ovf, _zer and _val operations from RPython. Kills +quite some code internally, and allows the JIT to do better +optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` +can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly +negative. + +.. branch: cpyext-old-buffers + +Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap + +.. branch: numpy-includes + +Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy +This allows building upstream numpy and scipy in pypy via cpyext + +.. branch: traceviewer-common-merge-point-formats + +Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. + +.. branch: cpyext-pickle + +Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch +at cpyext import time diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst @@ -0,0 +1,10 @@ +================================= +What's new in PyPy3 5.1.1 alpha 1 +================================= + +.. A recent revision, ignoring all other branches for this release +.. startrev: 29d14733e007 + +.. branch: py3.3 + +Python 3.3 compatibility diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -238,6 +238,15 @@ for use. The release packaging script will pick up the tcltk runtime in the lib directory and put it in the archive. +The lzma compression library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python 3.3 ship with CFFI wrappers for the lzma library, which can be +downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version +5.0.5, a prebuilt version can be downloaded from +http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature +http://tukaani.org/xz/xz-5.0.5-windows.zip.sig + Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -72,6 +72,11 @@ print('Goodbye2') # should not be reached """) +script_with_future = getscript(""" + from __future__ import division + from __future__ import print_function + """) + @contextmanager def setpythonpath(): @@ -441,6 +446,31 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_future_in_executed_script(self): + child = self.spawn(['-i', script_with_future]) + child.expect('>>> ') + child.sendline('x=1; print(x/2, 3/4)') + child.expect('0.5 0.75') + + def test_future_in_python_startup(self, monkeypatch): + monkeypatch.setenv('PYTHONSTARTUP', script_with_future) + child = self.spawn([]) + child.expect('>>> ') + child.sendline('x=1; print(x/2, 3/4)') + child.expect('0.5 0.75') + + def test_future_in_cmd(self): + child = self.spawn(['-i', '-c', 'from __future__ import division']) + child.expect('>>> ') + child.sendline('x=1; x/2; 3/4') + child.expect('0.5') + child.expect('0.75') + + def test_cmd_co_name(self): + child = self.spawn(['-c', + 'import sys; print sys._getframe(0).f_code.co_name']) + child.expect('') + def test_ignore_python_inspect(self): os.environ['PYTHONINSPECT_'] = '1' try: diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -12,7 +12,8 @@ class TypeDef(object): - def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict): + def __init__(self, __name, __base=None, __total_ordering__=None, + __buffer=None, **rawdict): "NOT_RPYTHON: initialization-time only" self.name = __name if __base is None: @@ -22,6 +23,8 @@ else: bases = [__base] self.bases = bases + assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" + self.buffer = __buffer self.heaptype = False self.hasdict = '__dict__' in rawdict # no __del__: use an RPython _finalize_() method and register_finalizer diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py --- a/pypy/module/__pypy__/interp_intop.py +++ b/pypy/module/__pypy__/interp_intop.py @@ -2,6 +2,19 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib import jit + + +# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT, +# because now it expects only Python-style divisions, not the +# C-style divisions of these two ll operations + at jit.dont_look_inside +def _int_floordiv(n, m): + return llop.int_floordiv(lltype.Signed, n, m) + + at jit.dont_look_inside +def _int_mod(n, m): + return llop.int_mod(lltype.Signed, n, m) @unwrap_spec(n=int, m=int) @@ -18,11 +31,11 @@ @unwrap_spec(n=int, m=int) def int_floordiv(space, n, m): - return space.wrap(llop.int_floordiv(lltype.Signed, n, m)) + return space.wrap(_int_floordiv(n, m)) @unwrap_spec(n=int, m=int) def int_mod(space, n, m): - return space.wrap(llop.int_mod(lltype.Signed, n, m)) + return space.wrap(_int_mod(n, m)) @unwrap_spec(n=int, m=int) def int_lshift(space, n, m): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1784,3 +1784,9 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + + def test_FFIFunctionWrapper(self): + ffi, lib = self.prepare("void f(void);", "test_FFIFunctionWrapper", + "void f(void) { }") + assert lib.f.__get__(42) is lib.f + assert lib.f.__get__(42, int) is lib.f diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py --- a/pypy/module/_cffi_backend/wrapper.py +++ b/pypy/module/_cffi_backend/wrapper.py @@ -100,6 +100,11 @@ doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename) return space.wrap(doc) + def descr_get(self, space, w_obj, w_type=None): + # never bind anything, but a __get__ is still present so that + # pydoc displays useful information (namely, the __repr__) + return self + @jit.unroll_safe def prepare_args(space, rawfunctype, args_w, start_index): @@ -136,5 +141,6 @@ __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper), __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper), __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc), + __get__ = interp2app(W_FunctionWrapper.descr_get), ) W_FunctionWrapper.typedef.acceptable_as_base_class = False diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter import gateway from pypy.module.cpyext.state import State from pypy.module.cpyext import api @@ -14,6 +15,12 @@ def startup(self, space): space.fromcache(State).startup(space) + method = pypy.module.cpyext.typeobject.get_new_method_def(space) + w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) + space.appexec([space.type(w_obj)], """(methodtype): + from pickle import Pickler + Pickler.dispatch[methodtype] = Pickler.save_global + """) def register_atexit(self, function): if len(self.atexit_funcs) >= 32: @@ -64,6 +71,7 @@ import pypy.module.cpyext.pyfile import pypy.module.cpyext.pystrtod import pypy.module.cpyext.pytraceback +import pypy.module.cpyext.methodobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -161,12 +161,13 @@ if copy_numpy_headers: try: - dstdir.mkdir('numpy') + dstdir.mkdir('_numpypy') + dstdir.mkdir('_numpypy/numpy') except py.error.EEXIST: pass - numpy_dstdir = dstdir / 'numpy' + numpy_dstdir = dstdir / '_numpypy' / 'numpy' - numpy_include_dir = include_dir / 'numpy' + numpy_include_dir = include_dir / '_numpypy' / 'numpy' numpy_headers = numpy_include_dir.listdir('*.h') + numpy_include_dir.listdir('*.inl') _copy_header_files(numpy_headers, numpy_dstdir) diff --git a/pypy/module/cpyext/include/numpy/README b/pypy/module/cpyext/include/_numpypy/numpy/README rename from pypy/module/cpyext/include/numpy/README rename to pypy/module/cpyext/include/_numpypy/numpy/README diff --git a/pypy/module/cpyext/include/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h rename from pypy/module/cpyext/include/numpy/__multiarray_api.h rename to pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/_numpypy/numpy/arrayobject.h rename from pypy/module/cpyext/include/numpy/arrayobject.h rename to pypy/module/cpyext/include/_numpypy/numpy/arrayobject.h diff --git a/pypy/module/cpyext/include/numpy/ndarraytypes.h b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h rename from pypy/module/cpyext/include/numpy/ndarraytypes.h rename to pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h rename from pypy/module/cpyext/include/numpy/npy_3kcompat.h rename to pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h diff --git a/pypy/module/cpyext/include/numpy/npy_common.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h rename from pypy/module/cpyext/include/numpy/npy_common.h rename to pypy/module/cpyext/include/_numpypy/numpy/npy_common.h diff --git a/pypy/module/cpyext/include/numpy/old_defines.h b/pypy/module/cpyext/include/_numpypy/numpy/old_defines.h rename from pypy/module/cpyext/include/numpy/old_defines.h rename to pypy/module/cpyext/include/_numpypy/numpy/old_defines.h diff --git a/pypy/module/cpyext/include/cStringIO.h b/pypy/module/cpyext/include/cStringIO.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/cStringIO.h @@ -0,0 +1,73 @@ +#ifndef Py_CSTRINGIO_H +#define Py_CSTRINGIO_H +#ifdef __cplusplus +extern "C" { +#endif +/* + + This header provides access to cStringIO objects from C. + Functions are provided for calling cStringIO objects and + macros are provided for testing whether you have cStringIO + objects. + + Before calling any of the functions or macros, you must initialize + the routines with: + + PycString_IMPORT + + This would typically be done in your init function. + +*/ + +#define PycStringIO_CAPSULE_NAME "cStringIO.cStringIO_CAPI" + +#define PycString_IMPORT \ + PycStringIO = ((struct PycStringIO_CAPI*)PyCapsule_Import(\ + PycStringIO_CAPSULE_NAME, 0)) + +/* Basic functions to manipulate cStringIO objects from C */ + +static struct PycStringIO_CAPI { + + /* Read a string from an input object. If the last argument + is -1, the remainder will be read. + */ + int(*cread)(PyObject *, char **, Py_ssize_t); + + /* Read a line from an input object. Returns the length of the read + line as an int and a pointer inside the object buffer as char** (so + the caller doesn't have to provide its own buffer as destination). + */ + int(*creadline)(PyObject *, char **); + + /* Write a string to an output object*/ + int(*cwrite)(PyObject *, const char *, Py_ssize_t); + + /* Get the output object as a Python string (returns new reference). */ + PyObject *(*cgetvalue)(PyObject *); + + /* Create a new output object */ + PyObject *(*NewOutput)(int); + + /* Create an input object from a Python string + (copies the Python string reference). + */ + PyObject *(*NewInput)(PyObject *); + + /* The Python types for cStringIO input and output objects. + Note that you can do input on an output object. + */ + PyTypeObject *InputType, *OutputType; + +} *PycStringIO; + +/* These can be used to test if you have one */ +#define PycStringIO_InputCheck(O) \ + (0) /* Py_TYPE(O)==PycStringIO->InputType) */ +#define PycStringIO_OutputCheck(O) \ + (0) /* Py_TYPE(O)==PycStringIO->OutputType) */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CSTRINGIO_H */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -44,8 +44,8 @@ dealloc=cfunction_dealloc) def cfunction_attach(space, py_obj, w_obj): + assert isinstance(w_obj, W_PyCFunctionObject) py_func = rffi.cast(PyCFunctionObject, py_obj) - assert isinstance(w_obj, W_PyCFunctionObject) py_func.c_m_ml = w_obj.ml py_func.c_m_self = make_ref(space, w_obj.w_self) py_func.c_m_module = make_ref(space, w_obj.w_module) diff --git a/pypy/module/cpyext/src/ndarrayobject.c b/pypy/module/cpyext/src/ndarrayobject.c --- a/pypy/module/cpyext/src/ndarrayobject.c +++ b/pypy/module/cpyext/src/ndarrayobject.c @@ -1,7 +1,7 @@ #include "Python.h" #include "pypy_numpy.h" -#include "numpy/arrayobject.h" +#include "_numpypy/numpy/arrayobject.h" #include /* memset, memcpy */ void diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -252,12 +252,16 @@ lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') w_text = space.wrapbytes("text") - assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0 + ref = make_ref(space, w_text) + prev_refcnt = ref.c_ob_refcnt + assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0 + assert ref.c_ob_refcnt == prev_refcnt assert lenp[0] == 4 assert rffi.charp2str(bufp[0]) == 'text' lltype.free(bufp, flavor='raw') lltype.free(lenp, flavor='raw') + api.Py_DecRef(ref) def test_eq(self, space, api): assert 1 == api._PyBytes_Eq(space.wrapbytes("hello"), space.wrapbytes("hello")) diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -1,4 +1,5 @@ import py +import os from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from rpython.rtyper.lltypesystem import rffi, lltype @@ -238,8 +239,10 @@ except: skip('numpy not importable') else: - cls.w_numpy_include = cls.space.wrap([]) - + numpy_incl = os.path.abspath(os.path.dirname(__file__) + + '/../include/_numpypy') + assert os.path.exists(numpy_incl) + cls.w_numpy_include = cls.space.wrap([numpy_incl]) def test_ndarray_object_c(self): mod = self.import_extension('foo', [ diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -125,7 +125,8 @@ return None def issubtype_w(self, w_sub, w_type): - return w_sub is w_type + is_root(w_type) + return NonConstant(True) def isinstance_w(self, w_obj, w_tp): try: @@ -414,6 +415,10 @@ def warn(self, w_msg, w_warn_type): pass +def is_root(w_obj): + assert isinstance(w_obj, W_Root) +is_root.expecting = W_Root + class FloatObject(W_Root): tp = FakeSpace.w_float def __init__(self, floatval): diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -262,7 +262,7 @@ [i0] i1 = int_add(i0, 1) i2 = int_sub(i1, 10) - i3 = int_floordiv(i2, 100) + i3 = int_xor(i2, 100) i4 = int_mul(i1, 1000) jump(i4) """ @@ -298,7 +298,7 @@ [i0] i1 = int_add(i0, 1) i2 = int_sub(i1, 10) - i3 = int_floordiv(i2, 100) + i3 = int_xor(i2, 100) i4 = int_mul(i1, 1000) jump(i4) """ diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py --- a/pypy/module/pypyjit/test_pypy_c/test_shift.py +++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py @@ -47,26 +47,74 @@ res = 0 a = 0 while a < 300: - assert a >= 0 - assert 0 <= b <= 10 - res = a/b # ID: div + res1 = a/b # ID: div + res2 = a/2 # ID: shift + res3 = a/11 # ID: mul + res += res1 + res2 + res3 a += 1 return res # log = self.run(main, [3]) - assert log.result == 99 + assert log.result == main(3) loop, = log.loops_by_filename(self.filepath) - if sys.maxint == 2147483647: - SHIFT = 31 + assert loop.match_by_id('div', """ + i56 = int_eq(i48, %d) + i57 = int_and(i56, i37) + guard_false(i57, descr=...) + i1 = call_i(_, i48, i3, descr=...) + """ % (-sys.maxint-1,)) + assert loop.match_by_id('shift', """ + i1 = int_rshift(i2, 1) + """) + if sys.maxint > 2**32: + args = (63, -5030930201920786804, 3) else: - SHIFT = 63 - assert loop.match_by_id('div', """ - i10 = int_floordiv(i6, i7) - i11 = int_mul(i10, i7) - i12 = int_sub(i6, i11) - i14 = int_rshift(i12, %d) - i15 = int_add(i10, i14) - """ % SHIFT) + args = (31, -1171354717, 3) + assert loop.match_by_id('mul', """ + i2 = int_rshift(i1, %d) + i3 = int_xor(i1, i2) + i4 = uint_mul_high(i3, %d) + i5 = uint_rshift(i4, %d) + i6 = int_xor(i5, i2) + """ % args) + + def test_modulo_optimization(self): + def main(b): + res = 0 + a = 0 + while a < 300: + res1 = a%b # ID: mod + res2 = a%2 # ID: and + res3 = a%11 # ID: mul + res += res1 + res2 + res3 + a += 1 + return res + # + log = self.run(main, [3]) + assert log.result == main(3) + loop, = log.loops_by_filename(self.filepath) + assert loop.match_by_id('mod', """ + i56 = int_eq(i48, %d) + i57 = int_and(i56, i37) + guard_false(i57, descr=...) + i1 = call_i(_, i48, i3, descr=...) + """ % (-sys.maxint-1,)) + assert loop.match_by_id('and', """ + i1 = int_and(i2, 1) + """) + if sys.maxint > 2**32: + args = (63, -5030930201920786804, 3) + else: + args = (31, -1171354717, 3) + assert loop.match_by_id('mul', """ + i2 = int_rshift(i1, %d) + i3 = int_xor(i1, i2) + i4 = uint_mul_high(i3, %d) + i5 = uint_rshift(i4, %d) + i6 = int_xor(i5, i2) + i7 = int_mul(i6, 11) + i8 = int_sub(i1, i7) + """ % args) def test_division_to_rshift_allcases(self): """ diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -1,11 +1,6 @@ import sys from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC -if sys.maxint == 2147483647: - SHIFT = 31 -else: - SHIFT = 63 - # XXX review the descrs to replace some EF=5 with EF=4 (elidable) @@ -28,10 +23,7 @@ guard_true(i14, descr=...) guard_not_invalidated(descr=...) i16 = int_eq(i6, %d) - i15 = int_mod(i6, i10) - i17 = int_rshift(i15, %d) - i18 = int_and(i10, i17) - i19 = int_add(i15, i18) + i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=) i21 = int_lt(i19, 0) guard_false(i21, descr=...) i22 = int_ge(i19, i10) @@ -49,7 +41,7 @@ i34 = int_add(i6, 1) --TICK-- jump(..., descr=...) - """ % (-sys.maxint-1, SHIFT)) + """ % (-sys.maxint-1,)) def test_long(self): def main(n): @@ -62,19 +54,25 @@ log = self.run(main, [1100], import_site=True) assert log.result == main(1100) loop, = log.loops_by_filename(self.filepath) + if sys.maxint > 2**32: + args = (63, -3689348814741910323, 3) + else: + args = (31, -858993459, 3) assert loop.match(""" i11 = int_lt(i6, i7) guard_true(i11, descr=...) guard_not_invalidated(descr=...) i13 = int_eq(i6, %d) # value provided below - i15 = int_mod(i6, 10) - i17 = int_rshift(i15, %d) # value provided below - i18 = int_and(10, i17) - i19 = int_add(i15, i18) - i21 = int_lt(i19, 0) - guard_false(i21, descr=...) - i22 = int_ge(i19, 10) - guard_false(i22, descr=...) + + # "mod 10" block: + i79 = int_rshift(i6, %d) + i80 = int_xor(i6, i79) + i82 = uint_mul_high(i80, %d) + i84 = uint_rshift(i82, %d) + i85 = int_xor(i84, i79) + i87 = int_mul(i85, 10) + i19 = int_sub(i6, i87) + i23 = strgetitem(p10, i19) p25 = newstr(1) strsetitem(p25, 0, i23) @@ -89,7 +87,7 @@ guard_no_overflow(descr=...) --TICK-- jump(..., descr=...) - """ % (-sys.maxint-1, SHIFT)) + """ % ((-sys.maxint-1,)+args)) def test_str_mod(self): def main(n): diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -7,6 +7,7 @@ from rpython.rlib import rstring, runicode, rlocale, rfloat, jit from rpython.rlib.objectmodel import specialize from rpython.rlib.rfloat import copysign, formatd +from rpython.rlib.rarithmetic import r_uint, intmask @specialize.argtype(1) @@ -836,33 +837,37 @@ return s # This part is slow. negative = value < 0 - value = abs(value) + base = r_uint(base) + value = r_uint(value) + if negative: # change the sign on the unsigned number: otherwise, + value = -value # we'd risk overflow if value==-sys.maxint-1 + # buf = ["\0"] * (8 * 8 + 6) # Too much on 32 bit, but who cares? i = len(buf) - 1 while True: - div = value // base - mod = value - div * base - digit = abs(mod) + div = value // base # unsigned + mod = value - div * base # unsigned, always in range(0,base) + digit = intmask(mod) digit += ord("0") if digit < 10 else ord("a") - 10 buf[i] = chr(digit) - value = div + value = div # unsigned i -= 1 if not value: break - if base == 2: + if base == r_uint(2): buf[i] = "b" buf[i - 1] = "0" - elif base == 8: + elif base == r_uint(8): buf[i] = "o" buf[i - 1] = "0" - elif base == 16: + elif base == r_uint(16): buf[i] = "x" buf[i - 1] = "0" else: buf[i] = "#" - buf[i - 1] = chr(ord("0") + base % 10) - if base > 10: - buf[i - 2] = chr(ord("0") + base // 10) + buf[i - 1] = chr(ord("0") + intmask(base % r_uint(10))) + if base > r_uint(10): + buf[i - 2] = chr(ord("0") + intmask(base // r_uint(10))) i -= 1 i -= 1 if negative: diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py --- a/pypy/tool/release/force-builds.py +++ b/pypy/tool/release/force-builds.py @@ -19,16 +19,16 @@ BUILDERS = [ 'own-linux-x86-32', 'own-linux-x86-64', - 'own-linux-armhf', +# 'own-linux-armhf', 'own-win-x86-32', - 'own-linux-s390x-2', + 'own-linux-s390x', # 'own-macosx-x86-32', 'pypy-c-jit-linux-x86-32', 'pypy-c-jit-linux-x86-64', # 'pypy-c-jit-freebsd-9-x86-64', 'pypy-c-jit-macosx-x86-64', 'pypy-c-jit-win-x86-32', - 'pypy-c-jit-linux-s390x-2', + 'pypy-c-jit-linux-s390x', 'build-pypy-c-jit-linux-armhf-raring', 'build-pypy-c-jit-linux-armhf-raspbian', 'build-pypy-c-jit-linux-armel', diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -213,11 +213,6 @@ default=False), BoolOption("merge_if_blocks", "Merge if ... elif chains", cmdline="--if-block-merge", default=True), - BoolOption("raisingop2direct_call", - "Transform operations that can implicitly raise an " - "exception into calls to functions that explicitly " - "raise exceptions", - default=False, cmdline="--raisingop2direct_call"), BoolOption("mallocs", "Remove mallocs", default=True), BoolOption("constfold", "Constant propagation", default=True), diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,6 +1,5 @@ from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg -from rpython.jit.backend.arm import support from rpython.jit.backend.arm.arch import WORD, PC_OFFSET from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin @@ -17,17 +16,6 @@ sandboxsafe=True) -def binary_helper_call(name): - function = getattr(support, 'arm_%s' % name) - - def f(self, c=cond.AL): - """Generates a call to a helper function, takes its - arguments in r0 and r1, result is placed in r0""" - addr = rffi.cast(lltype.Signed, function) - self.BL(addr, c) - return f - - class AbstractARMBuilder(object): def __init__(self, arch_version=7): self.arch_version = arch_version @@ -348,10 +336,6 @@ self.write32(c << 28 | 0x157ff05f) - DIV = binary_helper_call('int_div') - MOD = binary_helper_call('int_mod') - UDIV = binary_helper_call('uint_div') - FMDRR = VMOV_cr # uh, there are synonyms? FMRRD = VMOV_rc diff --git a/rpython/jit/backend/arm/helper/assembler.py b/rpython/jit/backend/arm/helper/assembler.py --- a/rpython/jit/backend/arm/helper/assembler.py +++ b/rpython/jit/backend/arm/helper/assembler.py @@ -46,20 +46,6 @@ f.__name__ = 'emit_op_%s' % name return f -def gen_emit_op_by_helper_call(name, opname): - helper = getattr(InstrBuilder, opname) - def f(self, op, arglocs, regalloc, fcond): - assert fcond is not None - if op.type != 'v': - regs = r.caller_resp[1:] + [r.ip] - else: - regs = r.caller_resp - with saved_registers(self.mc, regs, r.caller_vfp_resp): - helper(self.mc, fcond) - return fcond - f.__name__ = 'emit_op_%s' % name - return f - def gen_emit_cmp_op(name, true_cond): def f(self, op, arglocs, regalloc, fcond): l0, l1, res = arglocs diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py --- a/rpython/jit/backend/arm/helper/regalloc.py +++ b/rpython/jit/backend/arm/helper/regalloc.py @@ -72,25 +72,6 @@ res = self.force_allocate_reg_or_cc(op) return [loc1, loc2, res] -def prepare_op_by_helper_call(name): - def f(self, op, fcond): - assert fcond is not None - a0 = op.getarg(0) - a1 = op.getarg(1) - arg1 = self.rm.make_sure_var_in_reg(a0, selected_reg=r.r0) - arg2 = self.rm.make_sure_var_in_reg(a1, selected_reg=r.r1) - assert arg1 == r.r0 - assert arg2 == r.r1 - if not isinstance(a0, Const) and self.stays_alive(a0): - self.force_spill_var(a0) - self.possibly_free_vars_for_op(op) - self.free_temp_vars() - self.after_call(op) - self.possibly_free_var(op) - return [] - f.__name__ = name - return f - def prepare_int_cmp(self, op, fcond): assert fcond is not None boxes = list(op.getarglist()) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -3,7 +3,7 @@ from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm import shift from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE -from rpython.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call, +from rpython.jit.backend.arm.helper.assembler import ( gen_emit_op_unary_cmp, gen_emit_op_ri, gen_emit_cmp_op, @@ -92,6 +92,11 @@ self.mc.MUL(res.value, reg1.value, reg2.value) return fcond + def emit_op_uint_mul_high(self, op, arglocs, regalloc, fcond): + reg1, reg2, res = arglocs + self.mc.UMULL(r.ip.value, res.value, reg1.value, reg2.value) + return fcond + def emit_op_int_force_ge_zero(self, op, arglocs, regalloc, fcond): arg, res = arglocs self.mc.CMP_ri(arg.value, 0) @@ -132,10 +137,6 @@ self.guard_success_cc = c.VC return fcond - emit_op_int_floordiv = gen_emit_op_by_helper_call('int_floordiv', 'DIV') - emit_op_int_mod = gen_emit_op_by_helper_call('int_mod', 'MOD') - emit_op_uint_floordiv = gen_emit_op_by_helper_call('uint_floordiv', 'UDIV') - emit_op_int_and = gen_emit_op_ri('int_and', 'AND') emit_op_int_or = gen_emit_op_ri('int_or', 'ORR') emit_op_int_xor = gen_emit_op_ri('int_xor', 'EOR') @@ -466,7 +467,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() return fcond def _genop_same_as(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -7,7 +7,7 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import locations from rpython.jit.backend.arm.locations import imm, get_fp_offset -from rpython.jit.backend.arm.helper.regalloc import (prepare_op_by_helper_call, +from rpython.jit.backend.arm.helper.regalloc import ( prepare_unary_cmp, prepare_op_ri, prepare_int_cmp, @@ -397,9 +397,9 @@ else: self.rm.force_spill_var(var) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.vfprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.vfprm.before_call(save_all_regs) def _sync_var(self, v): if v.type == FLOAT: @@ -467,6 +467,8 @@ self.possibly_free_var(op) return [reg1, reg2, res] + prepare_op_uint_mul_high = prepare_op_int_mul + def prepare_op_int_force_ge_zero(self, op, fcond): argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op, [op.getarg(0)]) @@ -478,10 +480,6 @@ resloc = self.force_allocate_reg(op) return [argloc, imm(numbytes), resloc] - prepare_op_int_floordiv = prepare_op_by_helper_call('int_floordiv') - prepare_op_int_mod = prepare_op_by_helper_call('int_mod') - prepare_op_uint_floordiv = prepare_op_by_helper_call('unit_floordiv') - prepare_op_int_and = prepare_op_ri('int_and') prepare_op_int_or = prepare_op_ri('int_or') prepare_op_int_xor = prepare_op_ri('int_xor') @@ -554,8 +552,7 @@ prepare_op_call_f = _prepare_op_call prepare_op_call_n = _prepare_op_call - def _prepare_call(self, op, force_store=[], save_all_regs=False, - first_arg_index=1): + def _prepare_call(self, op, save_all_regs=False, first_arg_index=1): args = [None] * (op.numargs() + 3) calldescr = op.getdescr() assert isinstance(calldescr, CallDescr) @@ -573,17 +570,27 @@ args[1] = imm(size) args[2] = sign_loc - args[0] = self._call(op, args, force_store, save_all_regs) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + + args[0] = self._call(op, args, gc_level) return args - def _call(self, op, arglocs, force_store=[], save_all_regs=False): - # spill variables that need to be saved around calls - self.vfprm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + def _call(self, op, arglocs, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 + self.vfprm.before_call(save_all_regs=save_all_regs) + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 + self.rm.before_call(save_all_regs=save_all_regs) resloc = self.after_call(op) return resloc @@ -1070,7 +1077,7 @@ def _prepare_op_call_assembler(self, op, fcond): locs = self.locs_for_call_assembler(op) tmploc = self.get_scratch_reg(INT, selected_reg=r.r0) - resloc = self._call(op, locs + [tmploc], save_all_regs=True) + resloc = self._call(op, locs + [tmploc], gc_level=2) return locs + [resloc, tmploc] prepare_op_call_assembler_i = _prepare_op_call_assembler diff --git a/rpython/jit/backend/arm/support.py b/rpython/jit/backend/arm/support.py deleted file mode 100644 --- a/rpython/jit/backend/arm/support.py +++ /dev/null @@ -1,54 +0,0 @@ -from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.rlib.rarithmetic import r_uint -from rpython.translator.tool.cbuild import ExternalCompilationInfo - -eci = ExternalCompilationInfo(post_include_bits=[""" -static int pypy__arm_int_div(int a, int b) { - return a/b; -} -static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { - return a/b; -} -static int pypy__arm_int_mod(int a, int b) { - return a % b; -} -"""]) - - -def arm_int_div_emulator(a, b): - return int(a / float(b)) -arm_int_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) -arm_int_div = rffi.llexternal( - "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_uint_div_emulator(a, b): - return r_uint(a) / r_uint(b) -arm_uint_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) -arm_uint_div = rffi.llexternal( - "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, - _callable=arm_uint_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_int_mod_emulator(a, b): - sign = 1 - if a < 0: - a = -1 * a - sign = -1 - if b < 0: - b = -1 * b - res = a % b - return sign * res -arm_int_mod_sign = arm_int_div_sign -arm_int_mod = rffi.llexternal( - "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_mod_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) diff --git a/rpython/jit/backend/arm/test/test_arch.py b/rpython/jit/backend/arm/test/test_arch.py deleted file mode 100644 --- a/rpython/jit/backend/arm/test/test_arch.py +++ /dev/null @@ -1,23 +0,0 @@ -from rpython.jit.backend.arm import support - -def test_mod(): - assert support.arm_int_mod(10, 2) == 0 - assert support.arm_int_mod(11, 2) == 1 - assert support.arm_int_mod(11, 3) == 2 - -def test_mod2(): - assert support.arm_int_mod(-10, 2) == 0 - assert support.arm_int_mod(-11, 2) == -1 - assert support.arm_int_mod(-11, 3) == -2 - -def test_mod3(): - assert support.arm_int_mod(10, -2) == 0 - assert support.arm_int_mod(11, -2) == 1 - assert support.arm_int_mod(11, -3) == 2 - - -def test_div(): - assert support.arm_int_div(-7, 2) == -3 - assert support.arm_int_div(9, 2) == 4 - assert support.arm_int_div(10, 5) == 2 - diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py --- a/rpython/jit/backend/arm/test/test_assembler.py +++ b/rpython/jit/backend/arm/test/test_assembler.py @@ -193,32 +193,6 @@ self.a.gen_func_epilog() assert run_asm(self.a) == 61 - def test_DIV(self): - self.a.gen_func_prolog() - self.a.mc.MOV_ri(r.r0.value, 123) - self.a.mc.MOV_ri(r.r1.value, 2) - self.a.mc.DIV() - self.a.gen_func_epilog() - assert run_asm(self.a) == 61 - - def test_DIV2(self): - self.a.gen_func_prolog() - self.a.mc.gen_load_int(r.r0.value, -110) - self.a.mc.gen_load_int(r.r1.value, 3) - self.a.mc.DIV() - self.a.gen_func_epilog() - assert run_asm(self.a) == -36 - - def test_DIV3(self): - self.a.gen_func_prolog() - self.a.mc.gen_load_int(r.r8.value, 110) - self.a.mc.gen_load_int(r.r9.value, -3) - self.a.mc.MOV_rr(r.r0.value, r.r8.value) - self.a.mc.MOV_rr(r.r1.value, r.r9.value) - self.a.mc.DIV() - self.a.gen_func_epilog() - assert run_asm(self.a) == -36 - def test_bl_with_conditional_exec(self): functype = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) call_addr = rffi.cast(lltype.Signed, llhelper(functype, callme)) diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -574,27 +574,113 @@ self.assembler.regalloc_mov(reg, to) # otherwise it's clean + def _bc_spill(self, v, new_free_regs): + self._sync_var(v) + new_free_regs.append(self.reg_bindings.pop(v)) + def before_call(self, force_store=[], save_all_regs=0): - """ Spill registers before a call, as described by - 'self.save_around_call_regs'. Registers are not spilled if - they don't survive past the current operation, unless they - are listed in 'force_store'. 'save_all_regs' can be 0 (default), - 1 (save all), or 2 (save default+PTRs). + """Spill or move some registers before a call. By default, + this means: for every register in 'self.save_around_call_regs', + if there is a variable there and it survives longer than + the current operation, then it is spilled/moved somewhere else. + + 'save_all_regs' can be 0 (default set of registers), 1 (do that + for all registers), or 2 (default + gc ptrs). + + Overview of what we do (the implementation does it differently, + for the same result): + + * we first check the set of registers that are free: call it F. + + * possibly_free_vars() is implied for all variables (except + the ones listed in force_store): if they don't survive past + the current operation, they are forgotten now. (Their + register remain not in F, because they are typically + arguments to the call, so they should not be overwritten by + the next step.) + + * then for every variable that needs to be spilled/moved: if + there is an entry in F that is acceptable, pick it and emit a + move. Otherwise, emit a spill. Start doing this with the + variables that survive the shortest time, to give them a + better change to remain in a register---similar algo as + _pick_variable_to_spill(). + + Note: when a register is moved, it often (but not always) means + we could have been more clever and picked a better register in + the first place, when we did so earlier. It is done this way + anyway, as a local hack in this function, because on x86 CPUs + such register-register moves are almost free. """ + new_free_regs = [] + move_or_spill = [] + for v, reg in self.reg_bindings.items(): - if v not in force_store and self.longevity[v][1] <= self.position: + max_age = self.longevity[v][1] + if v not in force_store and max_age <= self.position: # variable dies del self.reg_bindings[v] - self.free_regs.append(reg) + new_free_regs.append(reg) continue - if save_all_regs != 1 and reg not in self.save_around_call_regs: - if save_all_regs == 0: - continue # we don't have to - if v.type != REF: - continue # only save GC pointers - self._sync_var(v) - del self.reg_bindings[v] - self.free_regs.append(reg) + + if save_all_regs == 1: + # we need to spill all registers in this mode + self._bc_spill(v, new_free_regs) + # + elif save_all_regs == 2 and v.type == REF: + # we need to spill all GC ptrs in this mode + self._bc_spill(v, new_free_regs) + # + elif reg not in self.save_around_call_regs: + continue # in a register like ebx/rbx: it is fine where it is + # + else: + # this is a register like eax/rax, which needs either + # spilling or moving. + move_or_spill.append((v, max_age)) + + if len(move_or_spill) > 0: + while len(self.free_regs) > 0: + new_reg = self.free_regs.pop() + if new_reg in self.save_around_call_regs: + new_free_regs.append(new_reg) # not this register... + continue + # This 'new_reg' is suitable for moving a candidate to. + # Pick the one with the smallest max_age. (This + # is one step of a naive sorting algo, slow in theory, + # but the list should always be very small so it + # doesn't matter.) + best_i = 0 + smallest_max_age = move_or_spill[0][1] + for i in range(1, len(move_or_spill)): + max_age = move_or_spill[i][1] + if max_age < smallest_max_age: + best_i = i + smallest_max_age = max_age + v, max_age = move_or_spill.pop(best_i) + # move from 'reg' to 'new_reg' + reg = self.reg_bindings[v] + if not we_are_translated(): + if move_or_spill: + assert max_age <= min([_a for _, _a in move_or_spill]) + assert reg in self.save_around_call_regs + assert new_reg not in self.save_around_call_regs + self.assembler.regalloc_mov(reg, new_reg) + self.reg_bindings[v] = new_reg # change the binding + new_free_regs.append(reg) + # + if len(move_or_spill) == 0: + break + else: + # no more free registers to move to, spill the rest + for v, max_age in move_or_spill: + self._bc_spill(v, new_free_regs) + + # re-add registers in 'new_free_regs', but in reverse order, + # so that the last ones (added just above, from + # save_around_call_regs) are picked last by future '.pop()' + while len(new_free_regs) > 0: + self.free_regs.append(new_free_regs.pop()) def after_call(self, v): """ Adjust registers according to the result of the call, diff --git a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py --- a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py @@ -496,22 +496,6 @@ self.interpret(ops, [s, ord('a')]) assert s[1] == 'a' - def test_division_optimized(self): - ops = ''' - [i7, i6] - label(i7, i6, descr=targettoken) - i18 = int_floordiv(i7, i6) - i19 = int_xor(i7, i6) - i21 = int_lt(i19, 0) - i22 = int_mod(i7, i6) - i23 = int_is_true(i22) - i24 = int_eq(i6, 4) - guard_false(i24) [i18] - jump(i18, i6, descr=targettoken) - ''' - self.interpret(ops, [10, 4]) - assert self.getint(0) == 2 - # FIXME: Verify that i19 - i23 are removed class TestRegallocFloats(BaseTestRegalloc): def setup_class(cls): diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -62,6 +62,12 @@ else: self.mc.mulld(res.value, l0.value, l1.value) + def emit_uint_mul_high(self, op, arglocs, regalloc): + l0, l1, res = arglocs + assert not l0.is_imm() + assert not l1.is_imm() + self.mc.mulhdu(res.value, l0.value, l1.value) + def do_emit_int_binary_ovf(self, op, arglocs): l0, l1, res = arglocs[0], arglocs[1], arglocs[2] self.mc.load_imm(r.SCRATCH, 0) @@ -80,24 +86,6 @@ else: self.mc.mulldox(*self.do_emit_int_binary_ovf(op, arglocs)) - def emit_int_floordiv(self, op, arglocs, regalloc): - l0, l1, res = arglocs - if IS_PPC_32: - self.mc.divw(res.value, l0.value, l1.value) - else: - self.mc.divd(res.value, l0.value, l1.value) - - def emit_int_mod(self, op, arglocs, regalloc): - l0, l1, res = arglocs - if IS_PPC_32: - self.mc.divw(r.r0.value, l0.value, l1.value) - self.mc.mullw(r.r0.value, r.r0.value, l1.value) - else: - self.mc.divd(r.r0.value, l0.value, l1.value) - self.mc.mulld(r.r0.value, r.r0.value, l1.value) - self.mc.subf(r.r0.value, r.r0.value, l0.value) - self.mc.mr(res.value, r.r0.value) - def emit_int_and(self, op, arglocs, regalloc): l0, l1, res = arglocs self.mc.and_(res.value, l0.value, l1.value) @@ -130,13 +118,6 @@ self.mc.srw(res.value, l0.value, l1.value) else: self.mc.srd(res.value, l0.value, l1.value) - - def emit_uint_floordiv(self, op, arglocs, regalloc): - l0, l1, res = arglocs - if IS_PPC_32: - self.mc.divwu(res.value, l0.value, l1.value) - else: - self.mc.divdu(res.value, l0.value, l1.value) emit_int_le = gen_emit_cmp_op(c.LE) emit_int_lt = gen_emit_cmp_op(c.LT) @@ -622,7 +603,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _genop_call(self, op, arglocs, regalloc): oopspecindex = regalloc.get_oopspecindex(op) diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -1,6 +1,7 @@ from rpython.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager, TempVar, compute_vars_longevity, BaseRegalloc) +from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32) from rpython.jit.codewriter import longlong from rpython.jit.backend.ppc.jump import (remap_frame_layout, @@ -369,9 +370,9 @@ # This operation is used only for testing self.force_spill_var(op.getarg(0)) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.fprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.fprm.before_call(save_all_regs) def after_call(self, v): if v.type == FLOAT: @@ -432,15 +433,13 @@ prepare_int_mul = helper.prepare_int_add_or_mul prepare_nursery_ptr_increment = prepare_int_add - prepare_int_floordiv = helper.prepare_binary_op - prepare_int_mod = helper.prepare_binary_op prepare_int_and = helper.prepare_binary_op prepare_int_or = helper.prepare_binary_op prepare_int_xor = helper.prepare_binary_op prepare_int_lshift = helper.prepare_binary_op prepare_int_rshift = helper.prepare_binary_op prepare_uint_rshift = helper.prepare_binary_op - prepare_uint_floordiv = helper.prepare_binary_op + prepare_uint_mul_high = helper.prepare_binary_op prepare_int_add_ovf = helper.prepare_binary_op prepare_int_sub_ovf = helper.prepare_binary_op @@ -758,7 +757,7 @@ src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(save_all_regs=False) + self._spill_before_call(gc_level=0) return [src_ptr_loc, dst_ptr_loc, src_ofs_loc, dst_ofs_loc, length_loc] @@ -791,13 +790,15 @@ prepare_call_f = _prepare_call prepare_call_n = _prepare_call - def _spill_before_call(self, save_all_regs=False): - # spill variables that need to be saved around calls + def _spill_before_call(self, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 self.fprm.before_call(save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) def _prepare_call(self, op, save_all_regs=False): @@ -805,7 +806,18 @@ args.append(None) for i in range(op.numargs()): args.append(self.loc(op.getarg(i))) - self._spill_before_call(save_all_regs) + + calldescr = op.getdescr() + assert isinstance(calldescr, CallDescr) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + self._spill_before_call(gc_level=gc_level) + if op.type != VOID: resloc = self.after_call(op) args[0] = resloc @@ -934,7 +946,7 @@ def _prepare_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._spill_before_call(save_all_regs=True) + self._spill_before_call(gc_level=2) if op.type != VOID: resloc = self.after_call(op) else: diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -532,6 +532,7 @@ rop.INT_AND, rop.INT_OR, rop.INT_XOR, + rop.UINT_MUL_HIGH, ]: OPERATIONS.append(BinaryOperation(_op)) @@ -548,8 +549,8 @@ ]: OPERATIONS.append(BinaryOperation(_op, boolres=True)) -OPERATIONS.append(BinaryOperation(rop.INT_FLOORDIV, ~3, 2)) -OPERATIONS.append(BinaryOperation(rop.INT_MOD, ~3, 2)) +#OPERATIONS.append(BinaryOperation(rop.INT_FLOORDIV, ~3, 2)) +#OPERATIONS.append(BinaryOperation(rop.INT_MOD, ~3, 2)) OPERATIONS.append(BinaryOperation(rop.INT_RSHIFT, LONG_BIT-1)) OPERATIONS.append(BinaryOperation(rop.INT_LSHIFT, LONG_BIT-1)) OPERATIONS.append(BinaryOperation(rop.UINT_RSHIFT, LONG_BIT-1)) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1289,6 +1289,9 @@ genop_float_mul = _binaryop('MULSD') genop_float_truediv = _binaryop('DIVSD') + def genop_uint_mul_high(self, op, arglocs, result_loc): + self.mc.MUL(arglocs[0]) + def genop_int_and(self, op, arglocs, result_loc): arg1 = arglocs[1] if IS_X86_64 and (isinstance(arg1, ImmedLoc) and @@ -1444,20 +1447,6 @@ self.mov(imm0, resloc) self.mc.CMOVNS(resloc, arglocs[0]) - def genop_int_mod(self, op, arglocs, resloc): - if IS_X86_32: - self.mc.CDQ() - elif IS_X86_64: - self.mc.CQO() - - self.mc.IDIV_r(ecx.value) - - genop_int_floordiv = genop_int_mod - - def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR_rr(edx.value, edx.value) - self.mc.DIV_r(ecx.value) - genop_llong_add = _binaryop("PADDQ") genop_llong_sub = _binaryop("PSUBQ") genop_llong_and = _binaryop("PAND") @@ -2123,7 +2112,11 @@ assert isinstance(saveerrloc, ImmedLoc) cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _store_force_index(self, guard_op): assert (guard_op.getopnum() == rop.GUARD_NOT_FORCED or diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -561,6 +561,27 @@ consider_int_sub_ovf = _consider_binop consider_int_add_ovf = _consider_binop_symm + def consider_uint_mul_high(self, op): + arg1, arg2 = op.getarglist() + # should support all cases, but is optimized for (box, const) + if isinstance(arg1, Const): + arg1, arg2 = arg2, arg1 + self.rm.make_sure_var_in_reg(arg2, selected_reg=eax) + l1 = self.loc(arg1) + # l1 is a register != eax, or stack_bp; or, just possibly, it + # can be == eax if arg1 is arg2 + assert not isinstance(l1, ImmedLoc) + assert l1 is not eax or arg1 is arg2 + # + # eax will be trash after the operation + self.rm.possibly_free_var(arg2) + tmpvar = TempVar() + self.rm.force_allocate_reg(tmpvar, selected_reg=eax) + self.rm.possibly_free_var(tmpvar) + # + self.rm.force_allocate_reg(op, selected_reg=edx) + self.perform(op, [l1], edx) + def consider_int_neg(self, op): res = self.rm.force_result_in_reg(op, op.getarg(0)) self.perform(op, [res], res) @@ -585,29 +606,6 @@ consider_int_rshift = consider_int_lshift consider_uint_rshift = consider_int_lshift - def _consider_int_div_or_mod(self, op, resultreg, trashreg): - l0 = self.rm.make_sure_var_in_reg(op.getarg(0), selected_reg=eax) - l1 = self.rm.make_sure_var_in_reg(op.getarg(1), selected_reg=ecx) - l2 = self.rm.force_allocate_reg(op, selected_reg=resultreg) - # the register (eax or edx) not holding what we are looking for - # will be just trash after that operation - tmpvar = TempVar() - self.rm.force_allocate_reg(tmpvar, selected_reg=trashreg) - assert l0 is eax - assert l1 is ecx - assert l2 is resultreg - self.rm.possibly_free_var(tmpvar) - - def consider_int_mod(self, op): - self._consider_int_div_or_mod(op, edx, eax) - self.perform(op, [eax, ecx], edx) - - def consider_int_floordiv(self, op): - self._consider_int_div_or_mod(op, eax, edx) - self.perform(op, [eax, ecx], eax) - - consider_uint_floordiv = consider_int_floordiv - def _consider_compop(self, op): vx = op.getarg(0) vy = op.getarg(1) @@ -797,22 +795,22 @@ else: self._consider_call(op) - def _call(self, op, arglocs, force_store=[], guard_not_forced=False): + def _call(self, op, arglocs, gc_level): # we need to save registers on the stack: # # - at least the non-callee-saved registers # - # - we assume that any call can collect, and we - # save also the callee-saved registers that contain GC pointers + # - if gc_level > 0, we save also the callee-saved registers that + # contain GC pointers # - # - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs - # anyway, in case we need to do cpu.force(). The issue is that - # grab_frame_values() would not be able to locate values in - # callee-saved registers. + # - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER. We + # have to save all regs anyway, in case we need to do + # cpu.force(). The issue is that grab_frame_values() would + # not be able to locate values in callee-saved registers. # - save_all_regs = guard_not_forced - self.xrm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: + save_all_regs = gc_level == 2 + self.xrm.before_call(save_all_regs=save_all_regs) + if gc_level == 1: gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap # we save all the registers for shadowstack and asmgcc for now # --- for asmgcc too: we can't say "register x is a gc ref" @@ -820,7 +818,7 @@ # more for now. if gcrootmap: # and gcrootmap.is_shadow_stack: save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + self.rm.before_call(save_all_regs=save_all_regs) if op.type != 'v': if op.type == FLOAT: resloc = self.xrm.after_call(op) @@ -840,9 +838,18 @@ sign_loc = imm1 else: sign_loc = imm0 + # + effectinfo = calldescr.get_extra_info() + if guard_not_forced: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + # self._call(op, [imm(size), sign_loc] + [self.loc(op.getarg(i)) for i in range(op.numargs())], - guard_not_forced=guard_not_forced) + gc_level=gc_level) def _consider_real_call(self, op): effectinfo = op.getdescr().get_extra_info() @@ -901,7 +908,7 @@ def _consider_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._call(op, locs, guard_not_forced=True) + self._call(op, locs, gc_level=2) consider_call_assembler_i = _consider_call_assembler consider_call_assembler_r = _consider_call_assembler consider_call_assembler_f = _consider_call_assembler diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -641,6 +641,7 @@ SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') NEG = _unaryop('NEG') + MUL = _unaryop('MUL') CMP = _binaryop('CMP') CMP16 = _binaryop('CMP16') diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -558,6 +558,9 @@ DIV_r = insn(rex_w, '\xF7', register(1), '\xF0') IDIV_r = insn(rex_w, '\xF7', register(1), '\xF8') + MUL_r = insn(rex_w, '\xF7', orbyte(4<<3), register(1), '\xC0') + MUL_b = insn(rex_w, '\xF7', orbyte(4<<3), stack_bp(1)) + IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0') IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2)) diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -14,6 +14,7 @@ from rpython.translator.backendopt.canraise import RaiseAnalyzer from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer from rpython.translator.backendopt.graphanalyze import DependencyTracker +from rpython.translator.backendopt.collectanalyze import CollectAnalyzer class CallControl(object): @@ -37,9 +38,9 @@ self.virtualizable_analyzer = VirtualizableAnalyzer(translator) self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator) self.randomeffects_analyzer = RandomEffectsAnalyzer(translator) - self.seen = DependencyTracker(self.readwrite_analyzer) - else: - self.seen = None + self.collect_analyzer = CollectAnalyzer(translator) + self.seen_rw = DependencyTracker(self.readwrite_analyzer) + self.seen_gc = DependencyTracker(self.collect_analyzer) # for index, jd in enumerate(jitdrivers_sd): jd.index = index @@ -294,9 +295,9 @@ "but the function has no result" % (op, )) # effectinfo = effectinfo_from_writeanalyze( - self.readwrite_analyzer.analyze(op, self.seen), self.cpu, + self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, - extradescr, + extradescr, self.collect_analyzer.analyze(op, self.seen_gc), ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -28,6 +28,11 @@ OS_THREADLOCALREF_GET = 5 # llop.threadlocalref_get OS_NOT_IN_TRACE = 8 # for calls not recorded in the jit trace # + OS_INT_PY_DIV = 12 # python signed division (neg. corrected) + OS_INT_UDIV = 13 # regular unsigned division + OS_INT_PY_MOD = 14 # python signed modulo (neg. corrected) + OS_INT_UMOD = 15 # regular unsigned modulo + # OS_STR_CONCAT = 22 # "stroruni.concat" OS_STR_SLICE = 23 # "stroruni.slice" OS_STR_EQUAL = 24 # "stroruni.equal" @@ -111,7 +116,8 @@ oopspecindex=OS_NONE, can_invalidate=False, call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET, - extradescrs=None): + extradescrs=None, + can_collect=True): readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields) readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays) readonly_descrs_interiorfields = frozenset_or_none( @@ -128,7 +134,8 @@ write_descrs_interiorfields, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + can_collect) tgt_func, tgt_saveerr = call_release_gil_target if tgt_func: key += (object(),) # don't care about caching in this case @@ -179,6 +186,7 @@ # result.extraeffect = extraeffect result.can_invalidate = can_invalidate + result.can_collect = can_collect result.oopspecindex = oopspecindex result.extradescrs = extradescrs From pypy.commits at gmail.com Thu Jun 2 17:19:48 2016 From: pypy.commits at gmail.com (raffael_t) Date: Thu, 02 Jun 2016 14:19:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Create other ops Message-ID: <5750a2f4.6322c20a.1882a.ffffe6e0@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84886:2e6c87aa86c2 Date: 2016-06-02 23:19 +0200 http://bitbucket.org/pypy/pypy/changeset/2e6c87aa86c2/ Log: Create other ops diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1324,8 +1324,29 @@ self.space.call_method(w_set, 'add', w_item) self.pushvalue(w_set) - #BUILD_SET_UNPACK (also as opcode) + def BUILD_SET_UNPACK(self, itemcount, next_instr): + w_sum = self.space.newset() + for i in range(itemcount): + self.space.updateset() #implement? + w_item = self.popvalue() + self.space.call_method(w_set, 'add', w_item) + self.pushvalue(w_set) + def BUILD_TUPLE_UNPACK(self, itemcount, next_instr): + w_sum = self.space.newset() + self.pushvalue(w_set) + + def BUILD_LIST_UNPACK(self, itemcount, next_instr): + w_sum = self.space.newset() + self.pushvalue(w_set) + + def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): + w_sum = self.space.newset() + self.pushvalue(w_set) + + def BUILD_MAP_UNPACK(self, itemcount, next_instr): + w_sum = self.space.newset() + self.pushvalue(w_set) ### ____________________________________________________________ ### class ExitFrame(Exception): From pypy.commits at gmail.com Thu Jun 2 17:21:48 2016 From: pypy.commits at gmail.com (gutworth) Date: Thu, 02 Jun 2016 14:21:48 -0700 (PDT) Subject: [pypy-commit] pypy default: remove useless call to a predicate Message-ID: <5750a36c.6a56c20a.f4dfe.ffffe840@mx.google.com> Author: Benjamin Peterson Branch: Changeset: r84887:bd8fffed3ba7 Date: 2016-06-02 14:20 -0700 http://bitbucket.org/pypy/pypy/changeset/bd8fffed3ba7/ Log: remove useless call to a predicate diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -564,7 +564,6 @@ self.emit_jump(ops.JUMP_FORWARD, end) self.use_next_block(next_except) self.emit_op(ops.END_FINALLY) # this END_FINALLY will always re-raise - self.is_dead_code() self.use_next_block(otherwise) self.visit_sequence(te.orelse) self.use_next_block(end) From pypy.commits at gmail.com Fri Jun 3 06:13:05 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 03:13:05 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: merge nonmovable-list into branch Message-ID: <57515831.4e0b1c0a.33053.00fa@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84888:8939df11b5d2 Date: 2016-06-02 18:33 +0300 http://bitbucket.org/pypy/pypy/changeset/8939df11b5d2/ Log: merge nonmovable-list into branch diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -184,7 +184,7 @@ def can_move(self, addr): return False - def malloc_fixedsize_nonmovable(self, typeid): + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): raise MemoryError def pin(self, addr): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -718,8 +718,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -628,8 +628,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -557,9 +557,10 @@ getfn(func, [SomeAddress()], annmodel.s_None) - self.malloc_nonmovable_ptr = getfn(GCClass.malloc_fixedsize_nonmovable, - [s_gc, s_typeid16], - s_gcref) + self.malloc_nonmovable_ptr = getfn( + GCClass.malloc_fixed_or_varsize_nonmovable, + [s_gc, s_typeid16, annmodel.SomeInteger()], + s_gcref) self.register_finalizer_ptr = getfn(GCClass.register_finalizer, [s_gc, @@ -800,12 +801,16 @@ c_has_light_finalizer = rmodel.inputconst(lltype.Bool, has_light_finalizer) + is_varsize = op.opname.endswith('_varsize') or flags.get('varsize') + if flags.get('nonmovable'): - assert op.opname == 'malloc' - assert not flags.get('varsize') + if not is_varsize: + v_length = rmodel.inputconst(lltype.Signed, 0) + else: + v_length = op.args[-1] malloc_ptr = self.malloc_nonmovable_ptr - args = [self.c_const_gc, c_type_id] - elif not op.opname.endswith('_varsize') and not flags.get('varsize'): + args = [self.c_const_gc, c_type_id, v_length] + elif not is_varsize: zero = flags.get('zero', False) if (self.malloc_fast_ptr is not None and not c_has_finalizer.value and diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1001,3 +1001,259 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.genop('gc_gettypeid', hop.args_v, resulttype=lltype.Signed) + +# ____________________________________________________________ + + +class _rawptr_missing_item(object): + pass +_rawptr_missing_item = _rawptr_missing_item() + + +class _ResizableListSupportingRawPtr(list): + """Calling this class is a no-op after translation. + + Before translation, it returns a new instance of + _ResizableListSupportingRawPtr, on which + rgc.nonmoving_raw_ptr_for_resizable_list() might be + used if needed. For now, only supports lists of chars. + """ + __slots__ = ('_raw_items',) # either None or a rffi.CCHARP + + def __init__(self, lst): + self._raw_items = None + self.__from_list(lst) + + def __resize(self): + """Called before an operation changes the size of the list""" + if self._raw_items is not None: + list.__init__(self, self.__as_list()) + self._raw_items = None + + def __from_list(self, lst): + """Initialize the list from a copy of the list 'lst'.""" + assert isinstance(lst, list) + for x in lst: + assert isinstance(x, str) and len(x) == 1 + if self is lst: + return + if len(self) != len(lst): + self.__resize() + if self._raw_items is None: + list.__init__(self, lst) + else: + assert len(self) == self._raw_items._obj.getlength() == len(lst) + for i in range(len(self)): + self._raw_items[i] = lst[i] + + def __as_list(self): + """Return a list (the same or a different one) which contains the + items in the regular way.""" + if self._raw_items is None: + return self + length = self._raw_items._obj.getlength() + assert length == len(self) + return [self._raw_items[i] for i in range(length)] + + def __getitem__(self, index): + if self._raw_items is None: + return list.__getitem__(self, index) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + return self._raw_items[index] + + def __setitem__(self, index, new): + if self._raw_items is None: + return list.__setitem__(self, index, new) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + self._raw_items[index] = new + + def __delitem__(self, index): + self.__resize() + list.__delitem__(self, index) + + def __getslice__(self, i, j): + return list.__getslice__(self.__as_list(), i, j) + + def __setslice__(self, i, j, new): + lst = self.__as_list() + list.__setslice__(lst, i, j, new) + self.__from_list(lst) + + def __delslice__(self, i, j): + lst = self.__as_list() + list.__delslice__(lst, i, j) + self.__from_list(lst) + + def __iter__(self): + try: + i = 0 + while True: + yield self[i] + i += 1 + except IndexError: + pass + + def __reversed__(self): + i = len(self) + while i > 0: + i -= 1 + yield self[i] + + def __contains__(self, item): + return list.__contains__(self.__as_list(), item) + + def __add__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(self.__as_list(), other) + + def __radd__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(other, self.__as_list()) + + def __iadd__(self, other): + self.__resize() + return list.__iadd__(self, other) + + def __eq__(self, other): + return list.__eq__(self.__as_list(), other) + def __ne__(self, other): + return list.__ne__(self.__as_list(), other) + def __ge__(self, other): + return list.__ge__(self.__as_list(), other) + def __gt__(self, other): + return list.__gt__(self.__as_list(), other) + def __le__(self, other): + return list.__le__(self.__as_list(), other) + def __lt__(self, other): + return list.__lt__(self.__as_list(), other) + + def __mul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __rmul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __imul__(self, other): + self.__resize() + return list.__imul__(self, other) + + def __repr__(self): + return '_ResizableListSupportingRawPtr(%s)' % ( + list.__repr__(self.__as_list()),) + + def append(self, object): + self.__resize() + return list.append(self, object) + + def count(self, value): + return list.count(self.__as_list(), value) + + def extend(self, iterable): + self.__resize() + return list.extend(self, iterable) + + def index(self, value, *start_stop): + return list.index(self.__as_list(), value, *start_stop) + + def insert(self, index, object): + self.__resize() + return list.insert(self, index, object) + + def pop(self, *opt_index): + self.__resize() + return list.pop(self, *opt_index) + + def remove(self, value): + self.__resize() + return list.remove(self, value) + + def reverse(self): + lst = self.__as_list() + list.reverse(lst) + self.__from_list(lst) + + def sort(self, *args, **kwds): + lst = self.__as_list() + list.sort(lst, *args, **kwds) + self.__from_list(lst) + + def _nonmoving_raw_ptr_for_resizable_list(self): + if self._raw_items is None: + existing_items = list(self) + from rpython.rtyper.lltypesystem import lltype, rffi + self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self), + flavor='raw', immortal=True) + self.__from_list(existing_items) + assert self._raw_items is not None + return self._raw_items + +def resizable_list_supporting_raw_ptr(lst): + return _ResizableListSupportingRawPtr(lst) + +def nonmoving_raw_ptr_for_resizable_list(lst): + assert isinstance(lst, _ResizableListSupportingRawPtr) + return lst._nonmoving_raw_ptr_for_resizable_list() + + +def _check_resizable_list_of_chars(s_list): + from rpython.annotator import model as annmodel + from rpython.rlib import debug + if annmodel.s_None.contains(s_list): + return # "None", will likely be generalized later + if not isinstance(s_list, annmodel.SomeList): + raise Exception("not a list, got %r" % (s_list,)) + if not isinstance(s_list.listdef.listitem.s_value, + (annmodel.SomeChar, annmodel.SomeImpossibleValue)): + raise debug.NotAListOfChars + s_list.listdef.resize() # must be resizable + +class Entry(ExtRegistryEntry): + _about_ = resizable_list_supporting_raw_ptr + + def compute_result_annotation(self, s_list): + _check_resizable_list_of_chars(s_list) + return s_list + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], 0) + +class Entry(ExtRegistryEntry): + _about_ = nonmoving_raw_ptr_for_resizable_list + + def compute_result_annotation(self, s_list): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.rtyper.llannotation import SomePtr + _check_resizable_list_of_chars(s_list) + return SomePtr(rffi.CCHARP) + + def specialize_call(self, hop): + v_list = hop.inputarg(hop.args_r[0], 0) + hop.exception_cannot_occur() # ignoring MemoryError + return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, + v_list) + +def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): + from rpython.rtyper.lltypesystem import lltype, rffi + array = ll_list.items + if can_move(array): + length = ll_list.length + new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length, + nonmovable=True) + i = 0 + while i < length: + new_array[i] = array[i] + i += 1 + ll_list.items = new_array + array = new_array + ptr = lltype.direct_arrayitems(array) + # ptr is a Ptr(FixedSizeArray(Char, 1)). Cast it to a rffi.CCHARP + return rffi.cast(rffi.CCHARP, ptr) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.test_llinterp import gengraph, interpret from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib import rgc # Force registration of gc.collect import gc import py, sys @@ -254,6 +254,153 @@ assert typer.custom_trace_funcs == [(TP, trace_func)] +def test_nonmoving_raw_ptr_for_resizable_list(): + def f(n): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst.append(chr(n)) + assert lst[3] == chr(n) + assert lst[-1] == chr(n) + # + ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst) + assert lst[:] == ['a', 'b', 'c', chr(n)] + assert lltype.typeOf(ptr) == rffi.CCHARP + assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)] + # + lst[-3] = 'X' + assert ptr[1] == 'X' + ptr[2] = 'Y' + assert lst[-2] == 'Y' + # + addr = rffi.cast(lltype.Signed, ptr) + ptr = rffi.cast(rffi.CCHARP, addr) + rgc.collect() # should not move lst.items + lst[-4] = 'g' + assert ptr[0] == 'g' + ptr[3] = 'H' + assert lst[-1] == 'H' + return lst + # + # direct untranslated run + lst = f(35) + assert isinstance(lst, rgc._ResizableListSupportingRawPtr) + # + # llinterp run + interpret(f, [35]) + # + # compilation with the GC transformer + import subprocess + from rpython.translator.interactive import Translation + # + def main(argv): + f(len(argv)) + print "OK!" + return 0 + # + t = Translation(main, gc="incminimark") + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') + +def test_ListSupportingRawPtr_direct(): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + + def check_nonresizing(): + assert lst[1] == lst[-2] == 'b' + lst[1] = 'X' + assert lst[1] == 'X' + lst[-1] = 'Y' + assert lst[1:3] == ['X', 'Y'] + assert lst[-2:9] == ['X', 'Y'] + lst[1:2] = 'B' + assert lst[:] == ['a', 'B', 'Y'] + assert list(iter(lst)) == ['a', 'B', 'Y'] + assert list(reversed(lst)) == ['Y', 'B', 'a'] + assert 'B' in lst + assert 'b' not in lst + assert p[0] == 'a' + assert p[1] == 'B' + assert p[2] == 'Y' + assert lst + ['*'] == ['a', 'B', 'Y', '*'] + assert ['*'] + lst == ['*', 'a', 'B', 'Y'] + assert lst + lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + base = ['8'] + base += lst + assert base == ['8', 'a', 'B', 'Y'] + assert lst == ['a', 'B', 'Y'] + assert ['a', 'B', 'Y'] == lst + assert ['a', 'B', 'Z'] != lst + assert ['a', 'B', 'Z'] > lst + assert ['a', 'B', 'Z'] >= lst + assert lst * 2 == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert 2 * lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert lst.count('B') == 1 + assert lst.index('Y') == 2 + lst.reverse() + assert lst == ['Y', 'B', 'a'] + lst.sort() + assert lst == ['B', 'Y', 'a'] + lst.sort(reverse=True) + assert lst == ['a', 'Y', 'B'] + lst[1] = 'b' + lst[2] = 'c' + assert list(lst) == ['a', 'b', 'c'] + + p = lst + check_nonresizing() + assert lst._raw_items is None + lst._nonmoving_raw_ptr_for_resizable_list() + p = lst._raw_items + check_nonresizing() + assert lst._raw_items == p + assert p[0] == 'a' + assert p[1] == 'b' + assert p[2] == 'c' + + def do_resizing_operation(): + del lst[1] + yield ['a', 'c'] + + lst[:2] = ['X'] + yield ['X', 'c'] + + del lst[:2] + yield ['c'] + + x = lst + x += ['t'] + yield ['a', 'b', 'c', 't'] + + x = lst + x *= 3 + yield ['a', 'b', 'c'] * 3 + + lst.append('f') + yield ['a', 'b', 'c', 'f'] + + lst.extend('fg') + yield ['a', 'b', 'c', 'f', 'g'] + + lst.insert(1, 'k') + yield ['a', 'k', 'b', 'c'] + + n = lst.pop(1) + assert n == 'b' + yield ['a', 'c'] + + lst.remove('c') + yield ['a', 'b'] + + assert lst == ['a', 'b', 'c'] + for expect in do_resizing_operation(): + assert lst == expect + assert lst._raw_items is None + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst._nonmoving_raw_ptr_for_resizable_list() # ____________________________________________________________ @@ -368,7 +515,6 @@ assert fq._triggered == 1 def test_finalizer_trigger_calls_too_much(self): - from rpython.rtyper.lltypesystem import lltype, rffi external_func = rffi.llexternal("foo", [], lltype.Void) # ^^^ with release_gil=True class X(object): diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -2174,7 +2174,8 @@ def malloc(T, n=None, flavor='gc', immortal=False, zero=False, - track_allocation=True, add_memory_pressure=False): + track_allocation=True, add_memory_pressure=False, + nonmovable=False): assert flavor in ('gc', 'raw') if zero or immortal: initialization = 'example' @@ -2200,7 +2201,8 @@ @analyzer_for(malloc) def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, - s_track_allocation=None, s_add_memory_pressure=None): + s_track_allocation=None, s_add_memory_pressure=None, + s_nonmovable=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2218,6 +2220,7 @@ assert s_track_allocation is None or s_track_allocation.is_constant() assert (s_add_memory_pressure is None or s_add_memory_pressure.is_constant()) + assert s_nonmovable is None or s_nonmovable.is_constant() # not sure how to call malloc() for the example 'p' in the # presence of s_extraargs r = SomePtr(Ptr(s_T.const)) diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -659,6 +659,7 @@ a[3] = 30 a[4] = 40 b0 = direct_arrayitems(a) + assert typeOf(b0) == Ptr(FixedSizeArray(Signed, 1)) b1 = direct_ptradd(b0, 1) b2 = direct_ptradd(b1, 1) b3 = direct_ptradd(b0, 3) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -348,7 +348,7 @@ @typer_for(lltype.malloc) def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, - i_add_memory_pressure=None): + i_add_memory_pressure=None, i_nonmovable=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' @@ -357,8 +357,10 @@ (i_flavor, lltype.Void), (i_zero, None), (i_track_allocation, None), - (i_add_memory_pressure, None)) - (v_flavor, v_zero, v_track_allocation, v_add_memory_pressure) = kwds_v + (i_add_memory_pressure, None), + (i_nonmovable, None)) + (v_flavor, v_zero, v_track_allocation, + v_add_memory_pressure, v_nonmovable) = kwds_v flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value @@ -368,6 +370,8 @@ flags['track_allocation'] = v_track_allocation.value if i_add_memory_pressure is not None: flags['add_memory_pressure'] = v_add_memory_pressure.value + if i_nonmovable is not None: + flags['nonmovable'] = v_nonmovable vlist.append(hop.inputconst(lltype.Void, flags)) assert 1 <= hop.nb_args <= 2 From pypy.commits at gmail.com Fri Jun 3 06:13:39 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 03:13:39 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: use resizable_list_supporting_raw_ptr in std/bytearrayobject, now passes some cpyext tests Message-ID: <57515853.59e61c0a.f3b88.ffff8082@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84889:078b429edc10 Date: 2016-06-03 09:35 +0300 http://bitbucket.org/pypy/pypy/changeset/078b429edc10/ Log: use resizable_list_supporting_raw_ptr in std/bytearrayobject, now passes some cpyext tests diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -17,12 +17,10 @@ # For the convenience of C programmers, the bytes type is considered # to contain a char pointer, not an unsigned char pointer. -# XXX The underlying data array is mutable, cpython gives direct access -# to ob_bytes as a RW pointer to bytes. How can we do this? -# One proposal is to make W_Bytearray.data into a nonmovable gc list -# as part of as_pyobj(), and expose data only through PyByteArray_AS_STRING -# Under this strategy ob_bytes could possibly not reflect the current state -# of the object +# Expose data as a rw cchar* only through PyByteArray_AsString +# Under this strategy the pointer could loose its synchronization with +# the underlying space.w_bytearray if PyByteArray_Resize is called, so +# hopefully the use of the pointer is short-lived PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) @@ -122,14 +120,3 @@ """Resize the internal buffer of bytearray to len.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AS_STRING(space, bytearray): - """Macro version of PyByteArray_AsString().""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_GET_SIZE(space, bytearray): - """Macro version of PyByteArray_Size().""" - raise NotImplementedError - - diff --git a/pypy/module/cpyext/include/bytearrayobject.h b/pypy/module/cpyext/include/bytearrayobject.h --- a/pypy/module/cpyext/include/bytearrayobject.h +++ b/pypy/module/cpyext/include/bytearrayobject.h @@ -17,6 +17,9 @@ * While CPython exposes interfaces to this object, pypy does not */ +#define PyByteArray_GET_SIZE(op) PyByteArray_Size((PyObject*)(op)) +#define PyByteArray_AS_STRING(op) PyByteArray_AsString((PyObject*)(op)) + /* Object layout */ typedef struct { PyObject_VAR_HEAD diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -6,6 +6,8 @@ from rpython.rlib.rstring import StringBuilder, ByteListBuilder from rpython.rlib.debug import check_list_of_chars from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr, + nonmoving_raw_ptr_for_resizable_list) from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -25,7 +27,7 @@ def __init__(self, data): check_list_of_chars(data) - self.data = data + self.data = resizable_list_supporting_raw_ptr(data) def __repr__(self): """representation for debugging purposes""" @@ -227,11 +229,11 @@ else: if count < 0: raise oefmt(space.w_ValueError, "bytearray negative count") - self.data = ['\0'] * count + self.data = resizable_list_supporting_raw_ptr(['\0'] * count) return data = makebytearraydata_w(space, w_source) - self.data = data + self.data = resizable_list_supporting_raw_ptr(data) def descr_repr(self, space): s = self.data @@ -1251,7 +1253,7 @@ self.data[start + i] = string[i] def get_raw_address(self): - return rffi.cast(rffi.CCHARP, 0) + return nonmoving_raw_ptr_for_resizable_list(self.data) @specialize.argtype(1) From pypy.commits at gmail.com Fri Jun 3 06:13:46 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 03:13:46 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: pass all tests, including resize Message-ID: <5751585a.4811c20a.7be2a.ffffc36f@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84890:a6e04b560dd1 Date: 2016-06-03 13:10 +0300 http://bitbucket.org/pypy/pypy/changeset/a6e04b560dd1/ Log: pass all tests, including resize diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -1,5 +1,6 @@ +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.objectmodel import specialize, we_are_translated from pypy.interpreter.error import OperationError, oefmt -from rpython.rtyper.lltypesystem import rffi, lltype from pypy.objspace.std.bytearrayobject import new_bytearray from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, build_type_checkers, @@ -74,29 +75,30 @@ #_______________________________________________________________________ - at cpython_api([PyObject], PyObject) -def PyByteArray_FromObject(space, o): + at cpython_api([PyObject], PyObject, result_is_ll=True) +def PyByteArray_FromObject(space, w_obj): """Return a new bytearray object from any object, o, that implements the buffer protocol. XXX expand about the buffer protocol, at least somewhere""" - raise NotImplementedError + w_buffer = space.call_function(space.w_bytearray, w_obj) + return make_ref(space, w_buffer) @cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True) def PyByteArray_FromStringAndSize(space, char_p, length): """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - s = rffi.charpsize2str(char_p, length) + w_s = space.wrap(rffi.charpsize2str(char_p, length)) else: - s = length - w_buffer = space.call_function(space.w_bytearray, space.wrap(s)) + w_s = space.wrap(length) + w_buffer = space.call_function(space.w_bytearray, w_s) return make_ref(space, w_buffer) @cpython_api([PyObject, PyObject], PyObject) -def PyByteArray_Concat(space, a, b): +def PyByteArray_Concat(space, w_left, w_right): """Concat bytearrays a and b and return a new bytearray with the result.""" - raise NotImplementedError + return space.call_method(w_left, '__add__', w_right) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyByteArray_Size(space, w_obj): @@ -116,7 +118,16 @@ "expected bytearray object, %T found", w_obj) @cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) -def PyByteArray_Resize(space, bytearray, len): +def PyByteArray_Resize(space, w_obj, newlen): """Resize the internal buffer of bytearray to len.""" - raise NotImplementedError - + if space.isinstance_w(w_obj, space.w_bytearray): + oldlen = space.len_w(w_obj) + if newlen > oldlen: + space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + elif oldlen > newlen: + assert newlen >= 0 + space.delitem(w_obj, space.wrap(slice(newlen, None, 1))) + return 0 + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -185,11 +185,15 @@ if (ret != 0) { printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); - return PyString_FromString("ret != 0"); + return NULL; } return ba; ''' )]) ret = module.bytearray_resize('abc', 6) assert len(ret) == 6,"%s, len=%d" % (ret, len(ret)) + assert ret == 'abc\x00\x00\x00' + ret = module.bytearray_resize('abcdefghi', 4) + assert len(ret) == 4,"%s, len=%d" % (ret, len(ret)) + assert ret == 'abcd' From pypy.commits at gmail.com Fri Jun 3 06:30:48 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 03:30:48 -0700 (PDT) Subject: [pypy-commit] pypy default: Tweak: this way seems more natural, and lets the annotator know that Message-ID: <57515c58.6a56c20a.f4dfe.ffffc98c@mx.google.com> Author: Armin Rigo Branch: Changeset: r84891:a0a038130dfc Date: 2016-06-03 12:31 +0200 http://bitbucket.org/pypy/pypy/changeset/a0a038130dfc/ Log: Tweak: this way seems more natural, and lets the annotator know that 'i' is never negative. diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -170,9 +170,8 @@ hexstring = hexstring.lower() data = [] length = len(hexstring) - i = -2 + i = 0 while True: - i += 2 while i < length and hexstring[i] == ' ': i += 1 if i >= length: @@ -187,6 +186,7 @@ if bot == -1: raise oefmt(space.w_ValueError, NON_HEX_MSG, i + 1) data.append(chr(top*16 + bot)) + i += 2 # in CPython bytearray.fromhex is a staticmethod, so # we ignore w_type and always return a bytearray From pypy.commits at gmail.com Fri Jun 3 06:50:42 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 03:50:42 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: rpython-ify (arigato) Message-ID: <57516102.e4b3c20a.35934.ffffd543@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84892:47dd7be1ca3f Date: 2016-06-03 13:48 +0300 http://bitbucket.org/pypy/pypy/changeset/47dd7be1ca3f/ Log: rpython-ify (arigato) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -98,7 +98,7 @@ @cpython_api([PyObject, PyObject], PyObject) def PyByteArray_Concat(space, w_left, w_right): """Concat bytearrays a and b and return a new bytearray with the result.""" - return space.call_method(w_left, '__add__', w_right) + return space.add(w_left, w_right) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyByteArray_Size(space, w_obj): @@ -126,7 +126,7 @@ space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 - space.delitem(w_obj, space.wrap(slice(newlen, None, 1))) + space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) return 0 else: raise oefmt(space.w_TypeError, diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -671,7 +671,7 @@ @cpython_api([PyObject, PyObject], PyObject) def PyUnicode_Concat(space, w_left, w_right): """Concat two strings giving a new Unicode string.""" - return space.call_method(w_left, '__add__', w_right) + return space.add(w_left, w_right) @cpython_api([rffi.CWCHARP, rffi.CWCHARP, Py_ssize_t], lltype.Void) def Py_UNICODE_COPY(space, target, source, length): From pypy.commits at gmail.com Fri Jun 3 06:50:44 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 03:50:44 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: merge default into cpyext-ext Message-ID: <57516104.832c1c0a.73c8c.ffff98dc@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84893:f6f66900d0d9 Date: 2016-06-03 13:49 +0300 http://bitbucket.org/pypy/pypy/changeset/f6f66900d0d9/ Log: merge default into cpyext-ext diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -125,4 +125,9 @@ .. branch: traceviewer-common-merge-point-formats -Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. \ No newline at end of file +Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. + +.. branch: cpyext-pickle + +Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch +at cpyext import time diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -564,7 +564,6 @@ self.emit_jump(ops.JUMP_FORWARD, end) self.use_next_block(next_except) self.emit_op(ops.END_FINALLY) # this END_FINALLY will always re-raise - self.is_dead_code() self.use_next_block(otherwise) self.visit_sequence(te.orelse) self.use_next_block(end) diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1784,3 +1784,9 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + + def test_FFIFunctionWrapper(self): + ffi, lib = self.prepare("void f(void);", "test_FFIFunctionWrapper", + "void f(void) { }") + assert lib.f.__get__(42) is lib.f + assert lib.f.__get__(42, int) is lib.f diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py --- a/pypy/module/_cffi_backend/wrapper.py +++ b/pypy/module/_cffi_backend/wrapper.py @@ -100,6 +100,11 @@ doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename) return space.wrap(doc) + def descr_get(self, space, w_obj, w_type=None): + # never bind anything, but a __get__ is still present so that + # pydoc displays useful information (namely, the __repr__) + return self + @jit.unroll_safe def prepare_args(space, rawfunctype, args_w, start_index): @@ -136,5 +141,6 @@ __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper), __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper), __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc), + __get__ = interp2app(W_FunctionWrapper.descr_get), ) W_FunctionWrapper.typedef.acceptable_as_base_class = False diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter import gateway from pypy.module.cpyext.state import State from pypy.module.cpyext import api @@ -14,6 +15,12 @@ def startup(self, space): space.fromcache(State).startup(space) + method = pypy.module.cpyext.typeobject.get_new_method_def(space) + w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap('')) + space.appexec([space.type(w_obj)], """(methodtype): + from pickle import Pickler + Pickler.dispatch[methodtype] = Pickler.save_global + """) def register_atexit(self, function): if len(self.atexit_funcs) >= 32: @@ -66,6 +73,7 @@ import pypy.module.cpyext.pyfile import pypy.module.cpyext.pystrtod import pypy.module.cpyext.pytraceback +import pypy.module.cpyext.methodobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -44,8 +44,8 @@ dealloc=cfunction_dealloc) def cfunction_attach(space, py_obj, w_obj): + assert isinstance(w_obj, W_PyCFunctionObject) py_func = rffi.cast(PyCFunctionObject, py_obj) - assert isinstance(w_obj, W_PyCFunctionObject) py_func.c_m_ml = w_obj.ml py_func.c_m_self = make_ref(space, w_obj.w_self) py_func.c_m_module = make_ref(space, w_obj.w_module) diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1502,7 +1502,7 @@ static PyObject * array_reduce(arrayobject *array) { - PyObject *dict, *result, *list; + PyObject *dict, *result, *list, *mod, *obj; dict = PyObject_GetAttrString((PyObject *)array, "__dict__"); if (dict == NULL) { @@ -1512,6 +1512,18 @@ dict = Py_None; Py_INCREF(dict); } + /* Return a tuple of (callable object, typecode, values, state) */ + mod = PyImport_ImportModule("array"); + if (mod == NULL) { + Py_DECREF(dict); + return NULL; + } + obj = PyObject_GetAttrString(mod, "_reconstruct"); + Py_DECREF(mod); + if (obj == NULL) { + Py_DECREF(dict); + return NULL; + } /* Unlike in Python 3.x, we never use the more efficient memory * representation of an array for pickling. This is unfortunately * necessary to allow array objects to be unpickled by Python 3.x, @@ -1524,7 +1536,7 @@ return NULL; } result = Py_BuildValue( - "O(cO)O", Py_TYPE(array), array->ob_descr->typecode, list, dict); + "O(cO)O", obj, array->ob_descr->typecode, list, dict); Py_DECREF(list); Py_DECREF(dict); return result; @@ -1916,6 +1928,11 @@ char c; PyObject *initial = NULL, *it = NULL; struct arraydescr *descr; + if (type == NULL) + { + /* when called from _reconstruct */ + type = &Arraytype; + } if (type == &Arraytype && !_PyArg_NoKeywords("array.array()", kwds)) return NULL; @@ -2017,6 +2034,11 @@ return NULL; } +static PyObject * +_reconstruct(PyTypeObject *type, PyObject *args) +{ + return array_new(type, args, NULL); +} PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ @@ -2223,6 +2245,7 @@ /* No functions in array module. */ static PyMethodDef a_methods[] = { + {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; @@ -2244,6 +2267,8 @@ return; Py_INCREF((PyObject *)&Arraytype); + if (PyType_Ready(&Arraytype) < 0) + return; PyModule_AddObject(m, "ArrayType", (PyObject *)&Arraytype); Py_INCREF((PyObject *)&Arraytype); PyModule_AddObject(m, "array", (PyObject *)&Arraytype); diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -67,3 +67,13 @@ '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0') + + def test_pickle(self): + import pickle + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + s = pickle.dumps(arr) + # pypy exports __dict__ on cpyext objects, so the pickle picks up the {} state value + #assert s == "carray\n_reconstruct\np0\n(S'i'\np1\n(lp2\nI1\naI2\naI3\naI4\natp3\nRp4\n." + rra = pickle.loads(s) # rra is arr backwards + #assert arr.tolist() == rra.tolist() diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -176,9 +176,8 @@ hexstring = hexstring.lower() data = [] length = len(hexstring) - i = -2 + i = 0 while True: - i += 2 while i < length and hexstring[i] == ' ': i += 1 if i >= length: @@ -193,6 +192,7 @@ if bot == -1: raise oefmt(space.w_ValueError, NON_HEX_MSG, i + 1) data.append(chr(top*16 + bot)) + i += 2 # in CPython bytearray.fromhex is a staticmethod, so # we ignore w_type and always return a bytearray diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py --- a/pypy/objspace/std/objectobject.py +++ b/pypy/objspace/std/objectobject.py @@ -180,7 +180,13 @@ if w_reduce is not None: w_cls = space.getattr(w_obj, space.wrap('__class__')) w_cls_reduce_meth = space.getattr(w_cls, w_st_reduce) - w_cls_reduce = space.getattr(w_cls_reduce_meth, space.wrap('im_func')) + try: + w_cls_reduce = space.getattr(w_cls_reduce_meth, space.wrap('im_func')) + except OperationError as e: + # i.e. PyCFunction from cpyext + if not e.match(space, space.w_AttributeError): + raise + w_cls_reduce = space.w_None w_objtype = space.w_object w_obj_dict = space.getattr(w_objtype, space.wrap('__dict__')) w_obj_reduce = space.getitem(w_obj_dict, w_st_reduce) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -467,7 +467,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() return fcond def _genop_same_as(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -397,9 +397,9 @@ else: self.rm.force_spill_var(var) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.vfprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.vfprm.before_call(save_all_regs) def _sync_var(self, v): if v.type == FLOAT: @@ -552,8 +552,7 @@ prepare_op_call_f = _prepare_op_call prepare_op_call_n = _prepare_op_call - def _prepare_call(self, op, force_store=[], save_all_regs=False, - first_arg_index=1): + def _prepare_call(self, op, save_all_regs=False, first_arg_index=1): args = [None] * (op.numargs() + 3) calldescr = op.getdescr() assert isinstance(calldescr, CallDescr) @@ -571,17 +570,27 @@ args[1] = imm(size) args[2] = sign_loc - args[0] = self._call(op, args, force_store, save_all_regs) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + + args[0] = self._call(op, args, gc_level) return args - def _call(self, op, arglocs, force_store=[], save_all_regs=False): - # spill variables that need to be saved around calls - self.vfprm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + def _call(self, op, arglocs, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 + self.vfprm.before_call(save_all_regs=save_all_regs) + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 + self.rm.before_call(save_all_regs=save_all_regs) resloc = self.after_call(op) return resloc @@ -1068,7 +1077,7 @@ def _prepare_op_call_assembler(self, op, fcond): locs = self.locs_for_call_assembler(op) tmploc = self.get_scratch_reg(INT, selected_reg=r.r0) - resloc = self._call(op, locs + [tmploc], save_all_regs=True) + resloc = self._call(op, locs + [tmploc], gc_level=2) return locs + [resloc, tmploc] prepare_op_call_assembler_i = _prepare_op_call_assembler diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -574,27 +574,113 @@ self.assembler.regalloc_mov(reg, to) # otherwise it's clean + def _bc_spill(self, v, new_free_regs): + self._sync_var(v) + new_free_regs.append(self.reg_bindings.pop(v)) + def before_call(self, force_store=[], save_all_regs=0): - """ Spill registers before a call, as described by - 'self.save_around_call_regs'. Registers are not spilled if - they don't survive past the current operation, unless they - are listed in 'force_store'. 'save_all_regs' can be 0 (default), - 1 (save all), or 2 (save default+PTRs). + """Spill or move some registers before a call. By default, + this means: for every register in 'self.save_around_call_regs', + if there is a variable there and it survives longer than + the current operation, then it is spilled/moved somewhere else. + + 'save_all_regs' can be 0 (default set of registers), 1 (do that + for all registers), or 2 (default + gc ptrs). + + Overview of what we do (the implementation does it differently, + for the same result): + + * we first check the set of registers that are free: call it F. + + * possibly_free_vars() is implied for all variables (except + the ones listed in force_store): if they don't survive past + the current operation, they are forgotten now. (Their + register remain not in F, because they are typically + arguments to the call, so they should not be overwritten by + the next step.) + + * then for every variable that needs to be spilled/moved: if + there is an entry in F that is acceptable, pick it and emit a + move. Otherwise, emit a spill. Start doing this with the + variables that survive the shortest time, to give them a + better change to remain in a register---similar algo as + _pick_variable_to_spill(). + + Note: when a register is moved, it often (but not always) means + we could have been more clever and picked a better register in + the first place, when we did so earlier. It is done this way + anyway, as a local hack in this function, because on x86 CPUs + such register-register moves are almost free. """ + new_free_regs = [] + move_or_spill = [] + for v, reg in self.reg_bindings.items(): - if v not in force_store and self.longevity[v][1] <= self.position: + max_age = self.longevity[v][1] + if v not in force_store and max_age <= self.position: # variable dies del self.reg_bindings[v] - self.free_regs.append(reg) + new_free_regs.append(reg) continue - if save_all_regs != 1 and reg not in self.save_around_call_regs: - if save_all_regs == 0: - continue # we don't have to - if v.type != REF: - continue # only save GC pointers - self._sync_var(v) - del self.reg_bindings[v] - self.free_regs.append(reg) + + if save_all_regs == 1: + # we need to spill all registers in this mode + self._bc_spill(v, new_free_regs) + # + elif save_all_regs == 2 and v.type == REF: + # we need to spill all GC ptrs in this mode + self._bc_spill(v, new_free_regs) + # + elif reg not in self.save_around_call_regs: + continue # in a register like ebx/rbx: it is fine where it is + # + else: + # this is a register like eax/rax, which needs either + # spilling or moving. + move_or_spill.append((v, max_age)) + + if len(move_or_spill) > 0: + while len(self.free_regs) > 0: + new_reg = self.free_regs.pop() + if new_reg in self.save_around_call_regs: + new_free_regs.append(new_reg) # not this register... + continue + # This 'new_reg' is suitable for moving a candidate to. + # Pick the one with the smallest max_age. (This + # is one step of a naive sorting algo, slow in theory, + # but the list should always be very small so it + # doesn't matter.) + best_i = 0 + smallest_max_age = move_or_spill[0][1] + for i in range(1, len(move_or_spill)): + max_age = move_or_spill[i][1] + if max_age < smallest_max_age: + best_i = i + smallest_max_age = max_age + v, max_age = move_or_spill.pop(best_i) + # move from 'reg' to 'new_reg' + reg = self.reg_bindings[v] + if not we_are_translated(): + if move_or_spill: + assert max_age <= min([_a for _, _a in move_or_spill]) + assert reg in self.save_around_call_regs + assert new_reg not in self.save_around_call_regs + self.assembler.regalloc_mov(reg, new_reg) + self.reg_bindings[v] = new_reg # change the binding + new_free_regs.append(reg) + # + if len(move_or_spill) == 0: + break + else: + # no more free registers to move to, spill the rest + for v, max_age in move_or_spill: + self._bc_spill(v, new_free_regs) + + # re-add registers in 'new_free_regs', but in reverse order, + # so that the last ones (added just above, from + # save_around_call_regs) are picked last by future '.pop()' + while len(new_free_regs) > 0: + self.free_regs.append(new_free_regs.pop()) def after_call(self, v): """ Adjust registers according to the result of the call, diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -603,7 +603,11 @@ assert saveerrloc.is_imm() cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _genop_call(self, op, arglocs, regalloc): oopspecindex = regalloc.get_oopspecindex(op) diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -1,6 +1,7 @@ from rpython.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager, TempVar, compute_vars_longevity, BaseRegalloc) +from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32) from rpython.jit.codewriter import longlong from rpython.jit.backend.ppc.jump import (remap_frame_layout, @@ -369,9 +370,9 @@ # This operation is used only for testing self.force_spill_var(op.getarg(0)) - def before_call(self, force_store=[], save_all_regs=False): - self.rm.before_call(force_store, save_all_regs) - self.fprm.before_call(force_store, save_all_regs) + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs) + self.fprm.before_call(save_all_regs) def after_call(self, v): if v.type == FLOAT: @@ -756,7 +757,7 @@ src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(save_all_regs=False) + self._spill_before_call(gc_level=0) return [src_ptr_loc, dst_ptr_loc, src_ofs_loc, dst_ofs_loc, length_loc] @@ -789,13 +790,15 @@ prepare_call_f = _prepare_call prepare_call_n = _prepare_call - def _spill_before_call(self, save_all_regs=False): - # spill variables that need to be saved around calls + def _spill_before_call(self, gc_level): + # spill variables that need to be saved around calls: + # gc_level == 0: callee cannot invoke the GC + # gc_level == 1: can invoke GC, save all regs that contain pointers + # gc_level == 2: can force, save all regs + save_all_regs = gc_level == 2 self.fprm.before_call(save_all_regs=save_all_regs) - if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - save_all_regs = 2 + if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap: + save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) def _prepare_call(self, op, save_all_regs=False): @@ -803,7 +806,18 @@ args.append(None) for i in range(op.numargs()): args.append(self.loc(op.getarg(i))) - self._spill_before_call(save_all_regs) + + calldescr = op.getdescr() + assert isinstance(calldescr, CallDescr) + effectinfo = calldescr.get_extra_info() + if save_all_regs: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + self._spill_before_call(gc_level=gc_level) + if op.type != VOID: resloc = self.after_call(op) args[0] = resloc @@ -932,7 +946,7 @@ def _prepare_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._spill_before_call(save_all_regs=True) + self._spill_before_call(gc_level=2) if op.type != VOID: resloc = self.after_call(op) else: diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2112,7 +2112,11 @@ assert isinstance(saveerrloc, ImmedLoc) cb.emit_call_release_gil(saveerrloc.value) else: - cb.emit() + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.check_can_collect(): + cb.emit() + else: + cb.emit_no_collect() def _store_force_index(self, guard_op): assert (guard_op.getopnum() == rop.GUARD_NOT_FORCED or diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -795,22 +795,22 @@ else: self._consider_call(op) - def _call(self, op, arglocs, force_store=[], guard_not_forced=False): + def _call(self, op, arglocs, gc_level): # we need to save registers on the stack: # # - at least the non-callee-saved registers # - # - we assume that any call can collect, and we - # save also the callee-saved registers that contain GC pointers + # - if gc_level > 0, we save also the callee-saved registers that + # contain GC pointers # - # - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs - # anyway, in case we need to do cpu.force(). The issue is that - # grab_frame_values() would not be able to locate values in - # callee-saved registers. + # - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER. We + # have to save all regs anyway, in case we need to do + # cpu.force(). The issue is that grab_frame_values() would + # not be able to locate values in callee-saved registers. # - save_all_regs = guard_not_forced - self.xrm.before_call(force_store, save_all_regs=save_all_regs) - if not save_all_regs: + save_all_regs = gc_level == 2 + self.xrm.before_call(save_all_regs=save_all_regs) + if gc_level == 1: gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap # we save all the registers for shadowstack and asmgcc for now # --- for asmgcc too: we can't say "register x is a gc ref" @@ -818,7 +818,7 @@ # more for now. if gcrootmap: # and gcrootmap.is_shadow_stack: save_all_regs = 2 - self.rm.before_call(force_store, save_all_regs=save_all_regs) + self.rm.before_call(save_all_regs=save_all_regs) if op.type != 'v': if op.type == FLOAT: resloc = self.xrm.after_call(op) @@ -838,9 +838,18 @@ sign_loc = imm1 else: sign_loc = imm0 + # + effectinfo = calldescr.get_extra_info() + if guard_not_forced: + gc_level = 2 + elif effectinfo is None or effectinfo.check_can_collect(): + gc_level = 1 + else: + gc_level = 0 + # self._call(op, [imm(size), sign_loc] + [self.loc(op.getarg(i)) for i in range(op.numargs())], - guard_not_forced=guard_not_forced) + gc_level=gc_level) def _consider_real_call(self, op): effectinfo = op.getdescr().get_extra_info() @@ -899,7 +908,7 @@ def _consider_call_assembler(self, op): locs = self.locs_for_call_assembler(op) - self._call(op, locs, guard_not_forced=True) + self._call(op, locs, gc_level=2) consider_call_assembler_i = _consider_call_assembler consider_call_assembler_r = _consider_call_assembler consider_call_assembler_f = _consider_call_assembler diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -14,6 +14,7 @@ from rpython.translator.backendopt.canraise import RaiseAnalyzer from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer from rpython.translator.backendopt.graphanalyze import DependencyTracker +from rpython.translator.backendopt.collectanalyze import CollectAnalyzer class CallControl(object): @@ -37,9 +38,9 @@ self.virtualizable_analyzer = VirtualizableAnalyzer(translator) self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator) self.randomeffects_analyzer = RandomEffectsAnalyzer(translator) - self.seen = DependencyTracker(self.readwrite_analyzer) - else: - self.seen = None + self.collect_analyzer = CollectAnalyzer(translator) + self.seen_rw = DependencyTracker(self.readwrite_analyzer) + self.seen_gc = DependencyTracker(self.collect_analyzer) # for index, jd in enumerate(jitdrivers_sd): jd.index = index @@ -294,9 +295,9 @@ "but the function has no result" % (op, )) # effectinfo = effectinfo_from_writeanalyze( - self.readwrite_analyzer.analyze(op, self.seen), self.cpu, + self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, - extradescr, + extradescr, self.collect_analyzer.analyze(op, self.seen_gc), ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -116,7 +116,8 @@ oopspecindex=OS_NONE, can_invalidate=False, call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET, - extradescrs=None): + extradescrs=None, + can_collect=True): readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields) readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays) readonly_descrs_interiorfields = frozenset_or_none( @@ -133,7 +134,8 @@ write_descrs_interiorfields, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + can_collect) tgt_func, tgt_saveerr = call_release_gil_target if tgt_func: key += (object(),) # don't care about caching in this case @@ -184,6 +186,7 @@ # result.extraeffect = extraeffect result.can_invalidate = can_invalidate + result.can_collect = can_collect result.oopspecindex = oopspecindex result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target @@ -230,6 +233,9 @@ def check_can_invalidate(self): return self.can_invalidate + def check_can_collect(self): + return self.can_collect + def check_is_elidable(self): return (self.extraeffect == self.EF_ELIDABLE_CAN_RAISE or self.extraeffect == self.EF_ELIDABLE_OR_MEMORYERROR or @@ -268,7 +274,8 @@ can_invalidate=False, call_release_gil_target= EffectInfo._NO_CALL_RELEASE_GIL_TARGET, - extradescr=None): + extradescr=None, + can_collect=True): from rpython.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -343,6 +350,9 @@ else: assert 0 # + if extraeffect >= EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE: + can_collect = True + # return EffectInfo(readonly_descrs_fields, readonly_descrs_arrays, readonly_descrs_interiorfields, @@ -353,7 +363,8 @@ oopspecindex, can_invalidate, call_release_gil_target, - extradescr) + extradescr, + can_collect) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -334,3 +334,37 @@ assert call_op.opname == 'direct_call' with py.test.raises(Exception): call_descr = cc.getcalldescr(call_op) + +def test_can_or_cannot_collect(): + from rpython.jit.backend.llgraph.runner import LLGraphCPU + prebuilts = [[5], [6]] + l = [] + def f1(n): + if n > 1: + raise IndexError + return prebuilts[n] # cannot collect + f1._dont_inline_ = True + + def f2(n): + return [n] # can collect + f2._dont_inline_ = True + + def f(n): + a = f1(n) + b = f2(n) + return len(a) + len(b) + + rtyper = support.annotate(f, [1]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + [f_graph] = [x for x in res if x.func is f] + for index, expected in [ + (0, False), # f1() + (1, True), # f2() + (2, False), # len() + (3, False)]: # len() + call_op = f_graph.startblock.operations[index] + assert call_op.opname == 'direct_call' + call_descr = cc.getcalldescr(call_op) + assert call_descr.extrainfo.check_can_collect() == expected diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1358,11 +1358,14 @@ return cls.minimal_size_in_nursery def write_barrier(self, addr_struct): - if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS: + # see OP_GC_BIT in translator/c/gc.py + if llop.gc_bit(lltype.Signed, self.header(addr_struct), + GCFLAG_TRACK_YOUNG_PTRS): self.remember_young_pointer(addr_struct) def write_barrier_from_array(self, addr_array, index): - if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS: + if llop.gc_bit(lltype.Signed, self.header(addr_array), + GCFLAG_TRACK_YOUNG_PTRS): if self.card_page_indices > 0: self.remember_young_pointer_from_array2(addr_array, index) else: diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -5,7 +5,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper import rmodel, annlowlevel from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup -from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS, llop +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.memory import gctypelayout from rpython.memory.gctransform.log import log from rpython.memory.gctransform.support import get_rtti, ll_call_destructor @@ -14,7 +14,7 @@ from rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF, WEAKREFPTR from rpython.memory.gctypelayout import FIN_TRIGGER_FUNC, FIN_HANDLER_ARRAY from rpython.tool.sourcetools import func_with_new_name -from rpython.translator.backendopt import graphanalyze +from rpython.translator.backendopt.collectanalyze import CollectAnalyzer from rpython.translator.backendopt.finalizer import FinalizerAnalyzer from rpython.translator.backendopt.support import var_needsgc import types @@ -23,33 +23,6 @@ TYPE_ID = llgroup.HALFWORD -class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): - - def analyze_direct_call(self, graph, seen=None): - try: - func = graph.func - except AttributeError: - pass - else: - if getattr(func, '_gctransformer_hint_cannot_collect_', False): - return False - if getattr(func, '_gctransformer_hint_close_stack_', False): - return True - return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, - seen) - def analyze_external_call(self, funcobj, seen=None): - if funcobj.random_effects_on_gcobjs: - return True - return graphanalyze.BoolGraphAnalyzer.analyze_external_call( - self, funcobj, seen) - def analyze_simple_operation(self, op, graphinfo): - if op.opname in ('malloc', 'malloc_varsize'): - flags = op.args[1].value - return flags['flavor'] == 'gc' - else: - return (op.opname in LL_OPERATIONS and - LL_OPERATIONS[op.opname].canmallocgc) - def propagate_no_write_barrier_needed(result, block, mallocvars, collect_analyzer, entrymap, startindex=0): diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -213,6 +213,8 @@ return self_type if self_type in (bool, int, long): return other_type + if self_type is float or other_type is float: + return float if self_type.SIGNED == other_type.SIGNED: return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS)) raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type)) @@ -297,6 +299,7 @@ def _widen(self, other, value): """ if one argument is int or long, the other type wins. + if one argument is float, the result is float. otherwise, produce the largest class to hold the result. """ self_type = type(self) diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -18,11 +18,11 @@ class Test_r_int: def test__add__(self): - self.binary_test(lambda x, y: x + y) + self.binary_test(lambda x, y: x + y, includes_floats=True) def test__sub__(self): - self.binary_test(lambda x, y: x - y) + self.binary_test(lambda x, y: x - y, includes_floats=True) def test__mul__(self): - self.binary_test(lambda x, y: x * y) + self.binary_test(lambda x, y: x * y, includes_floats=True) x = 3; y = [2] assert x*y == r_int(x)*y assert y*x == y*r_int(x) @@ -58,12 +58,15 @@ cmp = f(r_int(arg)) assert res == cmp - def binary_test(self, f, rargs = None): + def binary_test(self, f, rargs=None, includes_floats=False): if not rargs: rargs = (-10, -1, 3, 55) + types_list = [(int, r_int), (r_int, int), (r_int, r_int)] + if includes_floats: + types_list += [(float, r_int), (r_int, float)] for larg in (-10, -1, 0, 3, 1234): for rarg in rargs: - for types in ((int, r_int), (r_int, int), (r_int, r_int)): + for types in types_list: res = f(larg, rarg) left, right = types cmp = f(left(larg), right(rarg)) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -470,6 +470,7 @@ 'gc_pin' : LLOp(canrun=True), 'gc_unpin' : LLOp(canrun=True), 'gc__is_pinned' : LLOp(canrun=True), + 'gc_bit' : LLOp(sideeffects=False, canrun=True), 'gc_get_rpy_roots' : LLOp(), 'gc_get_rpy_referents': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -1,3 +1,4 @@ +import random, sys from rpython.flowspace.operation import op from rpython.rlib import debug from rpython.rlib.rarithmetic import is_valid_int @@ -680,6 +681,11 @@ def op_gc_writebarrier(addr): pass +def op_gc_bit(hdr, bitmask): + if hdr.tid & bitmask: + return random.randrange(1, sys.maxint) + return 0 + def op_shrink_array(array, smallersize): return False diff --git a/rpython/translator/backendopt/collectanalyze.py b/rpython/translator/backendopt/collectanalyze.py new file mode 100644 --- /dev/null +++ b/rpython/translator/backendopt/collectanalyze.py @@ -0,0 +1,33 @@ +from rpython.translator.backendopt import graphanalyze +from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS + +# NB. tests are in rpython/memory/gctransform/test/test_framework.py + + +class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): + + def analyze_direct_call(self, graph, seen=None): + try: + func = graph.func + except AttributeError: + pass + else: + if getattr(func, '_gctransformer_hint_cannot_collect_', False): + return False + if getattr(func, '_gctransformer_hint_close_stack_', False): + return True + return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, + seen) + def analyze_external_call(self, funcobj, seen=None): + if funcobj.random_effects_on_gcobjs: + return True + return graphanalyze.BoolGraphAnalyzer.analyze_external_call( + self, funcobj, seen) + + def analyze_simple_operation(self, op, graphinfo): + if op.opname in ('malloc', 'malloc_varsize'): + flags = op.args[1].value + return flags['flavor'] == 'gc' + else: + return (op.opname in LL_OPERATIONS and + LL_OPERATIONS[op.opname].canmallocgc) diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -391,6 +391,34 @@ raise AssertionError(subopnum) return ' '.join(parts) + def OP_GC_BIT(self, funcgen, op): + # This is a two-arguments operation (x, y) where x is a + # pointer and y is a constant power of two. It returns 0 if + # "(*(Signed*)x) & y == 0", and non-zero if it is "== y". + # + # On x86-64, emitting this is better than emitting a load + # followed by an INT_AND for the case where y doesn't fit in + # 32 bits. I've seen situations where a register was wasted + # to contain the constant 2**32 throughout a complete messy + # function; the goal of this GC_BIT is to avoid that. + # + # Don't abuse, though. If you need to check several bits in + # sequence, then it's likely better to load the whole Signed + # first; using GC_BIT would result in multiple accesses to + # memory. + # + bitmask = op.args[1].value + assert bitmask > 0 and (bitmask & (bitmask - 1)) == 0 + offset = 0 + while bitmask >= 0x100: + offset += 1 + bitmask >>= 8 + if sys.byteorder == 'big': + offset = 'sizeof(Signed)-%s' % (offset+1) + return '%s = ((char *)%s)[%s] & %d;' % (funcgen.expr(op.result), + funcgen.expr(op.args[0]), + offset, bitmask) + class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy): def gettransformer(self, translator): From pypy.commits at gmail.com Fri Jun 3 07:18:44 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 04:18:44 -0700 (PDT) Subject: [pypy-commit] cffi default: From PyPy 5.2, bytearray buffers can fetch a raw pointer, so Message-ID: <57516794.aaf0c20a.96323.ffffe884@mx.google.com> Author: Armin Rigo Branch: Changeset: r2701:61e03368485c Date: 2016-06-03 13:08 +0200 http://bitbucket.org/cffi/cffi/changeset/61e03368485c/ Log: From PyPy 5.2, bytearray buffers can fetch a raw pointer, so there is no reason any more to prevent from_buffer(bytearray()). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5945,8 +5945,8 @@ if (PyBytes_Check(x) || PyUnicode_Check(x)) return 1; - if (PyByteArray_Check(x)) /* <= this one here for PyPy compatibility */ - return 1; + /* From PyPy 5.2, bytearray buffers can fetch a raw pointer, so + there is no reason any more to prevent from_buffer(bytearray()). */ return 0; } @@ -5958,8 +5958,7 @@ if (invalid_input_buffer_type(x)) { PyErr_SetString(PyExc_TypeError, "from_buffer() cannot return the address of the " - "raw string within a "STR_OR_BYTES" or unicode or " - "bytearray object"); + "raw string within a "STR_OR_BYTES" or unicode object"); return NULL; } diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3328,13 +3328,12 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] -def test_from_buffer_not_str_unicode_bytearray(): +def test_from_buffer_not_str_unicode(): BChar = new_primitive_type("char") BCharP = new_pointer_type(BChar) BCharA = new_array_type(BCharP, None) py.test.raises(TypeError, from_buffer, BCharA, b"foo") py.test.raises(TypeError, from_buffer, BCharA, u+"foo") - py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) try: from __builtin__ import buffer except ImportError: @@ -3342,16 +3341,27 @@ else: py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - buffer(bytearray(b"foo"))) try: from __builtin__ import memoryview except ImportError: pass else: py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - memoryview(bytearray(b"foo"))) + +def test_from_buffer_bytearray(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + assert typeof(p) is BCharA + assert len(p) == 3 + assert repr(p) == "" + assert p[2] == b"z" + p[2] = b"." + assert a[2] == ord(".") + a[2] = ord("?") + assert p[2] == b"?" def test_from_buffer_more_cases(): try: From pypy.commits at gmail.com Fri Jun 3 07:59:19 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 04:59:19 -0700 (PDT) Subject: [pypy-commit] cffi default: Slice assignment with bytearray as source Message-ID: <57517117.0249c20a.76cab.fffff0ff@mx.google.com> Author: Armin Rigo Branch: Changeset: r2702:4d6671385e08 Date: 2016-06-03 13:59 +0200 http://bitbucket.org/cffi/cffi/changeset/4d6671385e08/ Log: Slice assignment with bytearray as source diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2155,21 +2155,34 @@ } } - /* A fast path for [0:N] = b"somestring", which also adds - support for Python 3: otherwise, you get integers while enumerating - the string, and you can't set them to characters :-/ + /* A fast path for [0:N] = b"somestring" or bytearray, which + also adds support for Python 3: otherwise, you get integers while + enumerating the string, and you can't set them to characters :-/ */ - if (PyBytes_Check(v) && (ct->ct_flags & CT_PRIMITIVE_CHAR) - && itemsize == sizeof(char)) { - if (PyBytes_GET_SIZE(v) != length) { + if ((ct->ct_flags & CT_PRIMITIVE_CHAR) && itemsize == sizeof(char)) { + char *src; + Py_ssize_t srclen; + if (PyBytes_Check(v)) { + srclen = PyBytes_GET_SIZE(v); + src = PyBytes_AS_STRING(v); + } + else if (PyByteArray_Check(v)) { + srclen = PyByteArray_GET_SIZE(v); + src = PyByteArray_AS_STRING(v); + } + else + goto other_types; + + if (srclen != length) { PyErr_Format(PyExc_ValueError, "need a string of length %zd, got %zd", - length, PyBytes_GET_SIZE(v)); + length, srclen); return -1; } - memcpy(cdata, PyBytes_AS_STRING(v), length); + memcpy(cdata, src, length); return 0; } + other_types: it = PyObject_GetIter(v); if (it == NULL) diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -2697,10 +2697,12 @@ BCharArray = new_array_type( new_pointer_type(new_primitive_type("char")), None) py.test.raises(TypeError, newp, BCharArray, bytearray(b"foo")) - p = newp(BCharArray, 4) - buffer(p)[:] = bytearray(b"foo\x00") - assert len(p) == 4 - assert list(p) == [b"f", b"o", b"o", b"\x00"] + p = newp(BCharArray, 5) + buffer(p)[:] = bytearray(b"foo.\x00") + assert len(p) == 5 + assert list(p) == [b"f", b"o", b"o", b".", b"\x00"] + p[1:3] = bytearray(b"XY") + assert list(p) == [b"f", b"X", b"Y", b".", b"\x00"] # XXX hack if sys.version_info >= (3,): From pypy.commits at gmail.com Fri Jun 3 07:59:21 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 04:59:21 -0700 (PDT) Subject: [pypy-commit] cffi default: Document 61e03368485c and 4d6671385e08. Remove the "new in version 0.9" Message-ID: <57517119.06921c0a.4eb28.ffffc3a4@mx.google.com> Author: Armin Rigo Branch: Changeset: r2703:61aa9f30a8cc Date: 2016-06-03 14:00 +0200 http://bitbucket.org/cffi/cffi/changeset/61aa9f30a8cc/ Log: Document 61e03368485c and 4d6671385e08. Remove the "new in version 0.9" as deprecated; keep only "new in version 1.x". diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -183,11 +183,11 @@ ``c_f = ffi.cast("FILE *", fileobj)`` and then pass around ``c_f``. * all `common Windows types`_ are defined if you run - on Windows (``DWORD``, ``LPARAM``, etc.). *Changed in version 0.9:* the - types ``TBYTE TCHAR LPCTSTR PCTSTR LPTSTR PTSTR PTBYTE PTCHAR`` are no - longer automatically defined; see `ffi.set_unicode()`_. + on Windows (``DWORD``, ``LPARAM``, etc.). Exception: + ``TBYTE TCHAR LPCTSTR PCTSTR LPTSTR PTSTR PTBYTE PTCHAR`` are + not automatically defined; see `ffi.set_unicode()`_. -* *New in version 0.9.3:* the other standard integer types from +* the other standard integer types from stdint.h, like ``intmax_t``, as long as they map to integers of 1, 2, 4 or 8 bytes. Larger integers are not supported. @@ -253,7 +253,7 @@ PTCHAR`` to be (pointers to) ``wchar_t``. If ``enabled_flag`` is False, declare these types to be (pointers to) plain 8-bit characters. (These types are not predeclared at all if you don't call -``set_unicode()``.) *New in version 0.9.* +``set_unicode()``.) The reason behind this method is that a lot of standard functions have two versions, like ``MessageBoxA()`` and ``MessageBoxW()``. The @@ -724,9 +724,8 @@ * ``source_extension`` has the same meaning as in ``ffi.set_source()``. -* The optional ``flags`` argument has been added in version 0.9; - see ``man dlopen`` - (ignored on Windows). It defaults to ``ffi.RTLD_NOW``. (With +* The optional ``flags`` argument (ignored on Windows) defaults to + ``ffi.RTLD_NOW``; see ``man dlopen``. (With ``ffi.set_source()``, you would use ``sys.setdlopenflags()``.) * The optional ``relative_to`` argument is useful if you need to list diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -171,16 +171,24 @@ buffer interface. This is the opposite of ``ffi.buffer()``. It gives a reference to the existing data, not a copy; for this reason, and for PyPy compatibility, it does not work with the built-in -types str or unicode or bytearray (or buffers/memoryviews on them). +types str or unicode (or buffers/memoryviews on them). It is meant to be used on objects -containing large quantities of raw data, like ``array.array`` or numpy +containing large quantities of raw data, like bytearrays +or ``array.array`` or numpy arrays. It supports both the old buffer API (in Python 2.x) and the new memoryview API. Note that if you pass a read-only buffer object, you still get a regular ````; it is your responsibility not to write there if the original buffer doesn't expect you to. The original object is kept alive (and, in case of memoryview, locked) as long as the cdata object returned by -``ffi.from_buffer()`` is alive. *New in version 0.9.* +``ffi.from_buffer()`` is alive. + +A common use case is calling a C function with some ``char *`` that +points to the internal buffer of a Python object; for this case you +can directly pass ``ffi.from_buffer(python_buffer)`` as argument to +the call. + +*New in version 1.7:* the python_buffer can be a bytearray object. ffi.memmove() @@ -255,7 +263,6 @@ offset within the struct of the given field. Corresponds to ``offsetof()`` in C. -*New in version 0.9:* You can give several field names in case of nested structures. You can also give numeric values which correspond to array items, in case of a pointer or array type. For example, ``ffi.offsetof("int[5]", 2)`` diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -16,6 +16,16 @@ this would only return False for cdata objects of a pointer type when the pointer is NULL. +* bytearrays: ``ffi.from_buffer(bytearray-object)`` is now supported. + (The reason it was not supported was that it was hard to do in PyPy, + but it works since PyPy 5.2.) You can call C functions with a + ``char *`` argument from any buffer object, now including + bytearrays, by giving them ``ffi.from_buffer(..)`` as argument. + Additionally, you can now do ``p[0:length] = bytearray-object``. + The problem with this was that a iterating over bytearrays gives + *numbers* instead of *characters*. (Now it is implemented with just + a memcpy, of course, not actually iterating over the characters.) + v1.6 ==== From pypy.commits at gmail.com Fri Jun 3 08:11:02 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 05:11:02 -0700 (PDT) Subject: [pypy-commit] cffi default: Fix docstrings Message-ID: <575173d6.aaf0c20a.96323.fffffe6e@mx.google.com> Author: Armin Rigo Branch: Changeset: r2704:dcb9f97611a7 Date: 2016-06-03 14:12 +0200 http://bitbucket.org/cffi/cffi/changeset/dcb9f97611a7/ Log: Fix docstrings diff --git a/c/ffi_obj.c b/c/ffi_obj.c --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -707,7 +707,7 @@ PyDoc_STRVAR(ffi_from_buffer_doc, "Return a that points to the data of the given Python\n" "object, which must support the buffer interface. Note that this is\n" -"not meant to be used on the built-in types str, unicode, or bytearray\n" +"not meant to be used on the built-in types str or unicode\n" "(you can build 'char[]' arrays explicitly) but only on objects\n" "containing large quantities of raw data in some other format, like\n" "'array.array' or numpy arrays."); diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ From pypy.commits at gmail.com Fri Jun 3 08:13:37 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 05:13:37 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: skip test for now Message-ID: <57517471.d21b1c0a.3c1c.359b@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84894:450b95d0d3a9 Date: 2016-06-03 15:12 +0300 http://bitbucket.org/pypy/pypy/changeset/450b95d0d3a9/ Log: skip test for now diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -190,6 +190,7 @@ del x, y def test_metaclass_compatible2(self): + skip('fails even with -A, fooType has BASETYPE flag') # XXX FIX - must raise since fooType (which is a base type) # does not have flag Py_TPFLAGS_BASETYPE module = self.import_module(name='foo') From pypy.commits at gmail.com Fri Jun 3 08:17:22 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 05:17:22 -0700 (PDT) Subject: [pypy-commit] cffi default: Add a test for another use case Message-ID: <57517552.07ecc20a.ae79d.fffff70d@mx.google.com> Author: Armin Rigo Branch: Changeset: r2705:847bbc0297f8 Date: 2016-06-03 14:18 +0200 http://bitbucket.org/cffi/cffi/changeset/847bbc0297f8/ Log: Add a test for another use case diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -2704,6 +2704,13 @@ p[1:3] = bytearray(b"XY") assert list(p) == [b"f", b"X", b"Y", b".", b"\x00"] +def test_string_assignment_to_byte_array(): + BByteArray = new_array_type( + new_pointer_type(new_primitive_type("unsigned char")), None) + p = newp(BByteArray, 5) + p[0:3] = bytearray(b"XYZ") + assert list(p) == [ord("X"), ord("Y"), ord("Z"), 0, 0] + # XXX hack if sys.version_info >= (3,): try: From pypy.commits at gmail.com Fri Jun 3 08:20:50 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 05:20:50 -0700 (PDT) Subject: [pypy-commit] pypy default: overdue update of pypy version to 5.3 Message-ID: <57517622.c8eac20a.1ff61.fffff51f@mx.google.com> Author: Matti Picus Branch: Changeset: r84895:2f8622c992ec Date: 2016-06-03 15:20 +0300 http://bitbucket.org/pypy/pypy/changeset/2f8622c992ec/ Log: overdue update of pypy version to 5.3 diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.2.0-alpha0" -#define PYPY_VERSION_NUM 0x05020000 +#define PYPY_VERSION "5.3.0-alpha0" +#define PYPY_VERSION_NUM 0x05030000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 2, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Fri Jun 3 08:33:18 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 05:33:18 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: update to cffi/847bbc0297f8 Message-ID: <5751790e.541a1c0a.4f7e9.3ef2@mx.google.com> Author: Armin Rigo Branch: cpyext-ext Changeset: r84896:b16aa9a51698 Date: 2016-06-03 14:33 +0200 http://bitbucket.org/pypy/pypy/changeset/b16aa9a51698/ Log: update to cffi/847bbc0297f8 diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -814,7 +814,7 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) << 1); /* check that '%s.%s' is " + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' @@ -991,7 +991,7 @@ prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') prnt(' int n = (%s) <= 0;' % (name,)) - prnt(' *o = (unsigned long long)((%s) << 0);' + prnt(' *o = (unsigned long long)((%s) | 0);' ' /* check that %s is an integer */' % (name, name)) if check_value is not None: if check_value > 0: @@ -1250,7 +1250,7 @@ def _emit_bytecode_UnknownIntegerType(self, tp, index): s = ('_cffi_prim_int(sizeof(%s), (\n' - ' ((%s)-1) << 0 /* check that %s is an integer type */\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' ' ) <= 0)' % (tp.name, tp.name, tp.name)) self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -243,6 +243,9 @@ def unicode_w(self, space): self._typed_unwrap_error(space, "unicode") + def bytearray_list_of_chars_w(self, space): + self._typed_unwrap_error(space, "bytearray") + def int_w(self, space, allow_conversion=True): # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w # raises w_TypeError diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -225,20 +225,29 @@ rffi.c_memcpy(target, source, ctitemsize * length) return # - # A fast path for [0:N] = "somestring". + # A fast path for [0:N] = "somestring" or some bytearray. from pypy.module._cffi_backend import ctypeprim space = self.space - if (space.isinstance_w(w_value, space.w_str) and - isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)): - from rpython.rtyper.annlowlevel import llstr - from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw - value = space.str_w(w_value) - if len(value) != length: - raise oefmt(space.w_ValueError, - "need a string of length %d, got %d", - length, len(value)) - copy_string_to_raw(llstr(value), target, 0, length) - return + if isinstance(ctitem, ctypeprim.W_CTypePrimitive) and ctitem.size == 1: + if space.isinstance_w(w_value, space.w_str): + from rpython.rtyper.annlowlevel import llstr + from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw + value = space.str_w(w_value) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a string of length %d, got %d", + length, len(value)) + copy_string_to_raw(llstr(value), target, 0, length) + return + if space.isinstance_w(w_value, space.w_bytearray): + value = w_value.bytearray_list_of_chars_w(space) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a bytearray of length %d, got %d", + length, len(value)) + for i in range(length): + target[i] = value[i] + return # w_iter = space.iter(w_value) for i in range(length): diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -43,9 +43,6 @@ else: return 'NULL' - def is_char_ptr_or_array(self): - return False - def is_unichar_ptr_or_array(self): return False diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -29,9 +29,6 @@ self.ctitem = ctitem self.can_cast_anything = could_cast_anything and ctitem.cast_anything - def is_char_ptr_or_array(self): - return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar) - def is_unichar_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar) diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -347,7 +347,7 @@ """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is -not meant to be used on the built-in types str, unicode, or bytearray +not meant to be used on the built-in types str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays.""" diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2686,10 +2686,19 @@ BCharArray = new_array_type( new_pointer_type(new_primitive_type("char")), None) py.test.raises(TypeError, newp, BCharArray, bytearray(b"foo")) - p = newp(BCharArray, 4) - buffer(p)[:] = bytearray(b"foo\x00") - assert len(p) == 4 - assert list(p) == [b"f", b"o", b"o", b"\x00"] + p = newp(BCharArray, 5) + buffer(p)[:] = bytearray(b"foo.\x00") + assert len(p) == 5 + assert list(p) == [b"f", b"o", b"o", b".", b"\x00"] + p[1:3] = bytearray(b"XY") + assert list(p) == [b"f", b"X", b"Y", b".", b"\x00"] + +def test_string_assignment_to_byte_array(): + BByteArray = new_array_type( + new_pointer_type(new_primitive_type("unsigned char")), None) + p = newp(BByteArray, 5) + p[0:3] = bytearray(b"XYZ") + assert list(p) == [ord("X"), ord("Y"), ord("Z"), 0, 0] # XXX hack if sys.version_info >= (3,): @@ -3317,13 +3326,12 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] -def test_from_buffer_not_str_unicode_bytearray(): +def test_from_buffer_not_str_unicode(): BChar = new_primitive_type("char") BCharP = new_pointer_type(BChar) BCharA = new_array_type(BCharP, None) py.test.raises(TypeError, from_buffer, BCharA, b"foo") py.test.raises(TypeError, from_buffer, BCharA, u+"foo") - py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) try: from __builtin__ import buffer except ImportError: @@ -3331,16 +3339,27 @@ else: py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - buffer(bytearray(b"foo"))) try: from __builtin__ import memoryview except ImportError: pass else: py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - memoryview(bytearray(b"foo"))) + +def test_from_buffer_bytearray(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + assert typeof(p) is BCharA + assert len(p) == 3 + assert repr(p) == "" + assert p[2] == b"z" + p[2] = b"." + assert a[2] == ord(".") + a[2] = ord("?") + assert p[2] == b"?" def test_from_buffer_more_cases(): try: diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -45,6 +45,9 @@ def charbuf_w(self, space): return ''.join(self.data) + def bytearray_list_of_chars_w(self, space): + return self.data + def nonmovable_carray(self, space): return BytearrayBuffer(self.data, False).get_raw_address() From pypy.commits at gmail.com Fri Jun 3 08:46:09 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 05:46:09 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-inheritance: improve subclassing in cpyext Message-ID: <57517c11.2426c20a.4a6b9.0332@mx.google.com> Author: Matti Picus Branch: cpyext-inheritance Changeset: r84897:874f04dc53d4 Date: 2016-06-03 15:22 +0300 http://bitbucket.org/pypy/pypy/changeset/874f04dc53d4/ Log: improve subclassing in cpyext From pypy.commits at gmail.com Fri Jun 3 08:46:11 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 05:46:11 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-inheritance: test now passes -A, fails untranslated. Also skip tests based on applevel sys, not host sys Message-ID: <57517c13.879d1c0a.68aaa.25c8@mx.google.com> Author: Matti Picus Branch: cpyext-inheritance Changeset: r84898:f9ebd9c81327 Date: 2016-06-03 15:44 +0300 http://bitbucket.org/pypy/pypy/changeset/f9ebd9c81327/ Log: test now passes -A, fails untranslated. Also skip tests based on applevel sys, not host sys diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -223,6 +223,42 @@ foo_getseters, /*tp_getset*/ }; +static PyTypeObject footypeNoSub = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.fooNoSub", /*tp_name*/ + sizeof(fooobject), /*tp_size*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + foo_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + foo_call, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + (setattrofunc)foo_setattro, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + foo_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + foo_methods, /*tp_methods*/ + foo_members, /*tp_members*/ + foo_getseters, /*tp_getset*/ +}; + + + /* A type that inherits from 'unicode */ typedef struct { @@ -664,6 +700,8 @@ if (PyType_Ready(&footype) < 0) return; + if (PyType_Ready(&footypeNoSub) < 0) + return; if (PyType_Ready(&UnicodeSubtype) < 0) return; if (PyType_Ready(&UnicodeSubtype2) < 0) @@ -697,6 +735,8 @@ return; if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) return; + if (PyDict_SetItemString(d, "fooTypeNoSub", (PyObject *)&footypeNoSub) < 0) + return; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) return; if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -6,7 +6,6 @@ from pypy.module.cpyext.typeobject import PyTypeObjectPtr import pytest -import sys class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): @@ -124,8 +123,10 @@ obj = module.fooType.classmeth() assert obj is module.fooType - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython segfaults') def test_new(self): + import sys + if '__pypy__' not in sys.builtin_module_names: + skip('cpython segfaults') # XXX cpython segfaults but if run singly (with -k test_new) this passes module = self.import_module(name='foo') obj = module.new() @@ -196,11 +197,10 @@ del x, y def test_metaclass_compatible2(self): - skip('type.__new__ does not check acceptable_as_base_class') - # XXX FIX - must raise since fooType (which is a base type) + # must raise since fooTypeNoSub (which is a base type) # does not have flag Py_TPFLAGS_BASETYPE module = self.import_module(name='foo') - raises(TypeError, module.MetaType, 'other', (module.fooType,), {}) + raises(TypeError, module.MetaType, 'other', (module.fooTypeNoSub,), {}) def test_sre(self): import sys for m in ['_sre', 'sre_compile', 'sre_constants', 'sre_parse', 're']: @@ -901,8 +901,10 @@ #print('calling module.footype()...') module.footype("X", (object,), {}) - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython fails') def test_app_subclass_of_c_type(self): + import sys + if '__pypy__' not in sys.builtin_module_names: + skip('cpython adds 6 bytes to size') # on cpython, the size changes (6 bytes added) module = self.import_module(name='foo') size = module.size_of_instances(module.fooType) From pypy.commits at gmail.com Fri Jun 3 08:53:21 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 05:53:21 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: fix test - buffer(bytearray) succeeds Message-ID: <57517dc1.4674c20a.8a463.0a1d@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84899:a884eb833aa5 Date: 2016-06-03 15:52 +0300 http://bitbucket.org/pypy/pypy/changeset/a884eb833aa5/ Log: fix test - buffer(bytearray) succeeds diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py --- a/pypy/objspace/std/test/test_bufferobject.py +++ b/pypy/objspace/std/test/test_bufferobject.py @@ -201,5 +201,5 @@ def test_pypy_raw_address_base(self): raises(ValueError, buffer("foobar")._pypy_raw_address) raises(ValueError, buffer(u"foobar")._pypy_raw_address) - e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address) - assert 'BytearrayBuffer' in str(e.value) + a = buffer(bytearray("foobar"))._pypy_raw_address() + assert a != 0 From pypy.commits at gmail.com Fri Jun 3 08:58:49 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 05:58:49 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: Expose to the JIT the type dispatching part of _do_setslice(), at least Message-ID: <57517f09.08371c0a.8f5a7.4da0@mx.google.com> Author: Armin Rigo Branch: cpyext-ext Changeset: r84900:158c9efd5ffd Date: 2016-06-03 14:59 +0200 http://bitbucket.org/pypy/pypy/changeset/158c9efd5ffd/ Log: Expose to the JIT the type dispatching part of _do_setslice(), at least diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -245,10 +245,17 @@ raise oefmt(space.w_ValueError, "need a bytearray of length %d, got %d", length, len(value)) - for i in range(length): - target[i] = value[i] + self._copy_list_of_chars_to_raw(value, target, length) return # + self._do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length) + + @staticmethod + def _do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length): + # general case, contains a loop + # (XXX is it worth adding a jitdriver here?) w_iter = space.iter(w_value) for i in range(length): try: @@ -269,6 +276,12 @@ raise oefmt(space.w_ValueError, "got more than %d values to unpack", length) + @staticmethod + def _copy_list_of_chars_to_raw(value, target, length): + # contains a loop, moved out of _do_setslice() + for i in range(length): + target[i] = value[i] + def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) From pypy.commits at gmail.com Fri Jun 3 09:28:39 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 06:28:39 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: Translation fix. Also move another part of a function to be seen by the JIT. Message-ID: <57518607.e505c20a.6fe85.195a@mx.google.com> Author: Armin Rigo Branch: cpyext-ext Changeset: r84901:6cfa79ea0abf Date: 2016-06-03 15:26 +0200 http://bitbucket.org/pypy/pypy/changeset/6cfa79ea0abf/ Log: Translation fix. Also move another part of a function to be seen by the JIT. diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -222,7 +222,10 @@ w_value.get_array_length() == length): # fast path: copying from exactly the correct type with w_value as source: - rffi.c_memcpy(target, source, ctitemsize * length) + source = rffi.cast(rffi.VOIDP, source) + target = rffi.cast(rffi.VOIDP, target) + size = rffi.cast(rffi.SIZE_T, ctitemsize * length) + rffi.c_memcpy(target, source, size) return # # A fast path for [0:N] = "somestring" or some bytearray. diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -51,12 +51,8 @@ value = rffi.cast(rffi.CCHARP, value) return cdataobj.W_CData(space, value, self) - def _convert_array_from_listview(self, cdata, w_ob): - if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path - return - # + def _convert_array_from_listview(self, cdata, lst_w): space = self.space - lst_w = space.listview(w_ob) if self.length >= 0 and len(lst_w) > self.length: raise oefmt(space.w_IndexError, "too many initializers for '%s' (got %d)", @@ -70,7 +66,10 @@ space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): - self._convert_array_from_listview(cdata, w_ob) + if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path + pass + else: + self._convert_array_from_listview(cdata, space.listview(w_ob)) elif (self.can_cast_anything or (self.ctitem.is_primitive_integer and self.ctitem.size == rffi.sizeof(lltype.Char))): From pypy.commits at gmail.com Fri Jun 3 09:28:40 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 06:28:40 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: fix test, same as a884eb833aa5 Message-ID: <57518608.6322c20a.1882a.16b5@mx.google.com> Author: Armin Rigo Branch: cpyext-ext Changeset: r84902:686345719452 Date: 2016-06-03 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/686345719452/ Log: fix test, same as a884eb833aa5 diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -57,5 +57,5 @@ def test_pypy_raw_address_base(self): raises(ValueError, memoryview("foobar")._pypy_raw_address) - e = raises(ValueError, memoryview(bytearray("foobar"))._pypy_raw_address) - assert 'BytearrayBuffer' in str(e.value) + a = memoryview(bytearray("foobar"))._pypy_raw_address() + assert a != 0 From pypy.commits at gmail.com Fri Jun 3 10:38:12 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Jun 2016 07:38:12 -0700 (PDT) Subject: [pypy-commit] cffi default: Add a warning Message-ID: <57519654.d81b1c0a.325c2.fffff9bb@mx.google.com> Author: Armin Rigo Branch: Changeset: r2706:989baa0a3b77 Date: 2016-06-03 16:39 +0200 http://bitbucket.org/cffi/cffi/changeset/989baa0a3b77/ Log: Add a warning diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -189,6 +189,9 @@ the call. *New in version 1.7:* the python_buffer can be a bytearray object. +Be careful: if the bytearray gets resized (e.g. its ``.append()`` +method is called), then the ```` object will point to freed +memory and must not be used any more. ffi.memmove() From pypy.commits at gmail.com Fri Jun 3 10:49:18 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:18 -0700 (PDT) Subject: [pypy-commit] pypy default: draft release notice Message-ID: <575198ee.89cbc20a.154d3.3825@mx.google.com> Author: Matti Picus Branch: Changeset: r84903:6f129919a7aa Date: 2016-06-03 17:38 +0300 http://bitbucket.org/pypy/pypy/changeset/6f129919a7aa/ Log: draft release notice diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -0,0 +1,139 @@ +============ +PyPy2.7 v5.3 +============ + +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. +This release includes further improvements for the CAPI compatibility layer +which we call cpyext. In addtion to complete support for lxml, we now pass +most (more than 90%) of the upstream numpy test suite, and much of SciPy is +supported as well. + +We also improved the speed of ... and ... + +We updated cffi_ to ... + +You can download the PyPy2.7 v5.3 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy +.. _cffi: https://cffi.readthedocs.org +.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html +.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.1 released in April 2016) +========================================================= + +* New features: + + * Merge a major expansion of the C-API support in cpyext, here are some of + the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible + - fix many edge cases exposed by fixing tests to run with -A + - issequence() logic matches cpython + - make PyStringObject and PyUnicodeObject field names compatible with cpython + - add prelminary support for PyDateTime_* + - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, + PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, + - PyAnySet_CheckExact, PyUnicode_Concat + - improve support for PyGILState_Ensure, PyGILState_Release, and thread + primitives, also find a case where CPython will allow thread creation + before PyEval_InitThreads is run, dissallow on PyPy + - create a PyObject-specific list strategy + - rewrite slot assignment for typeobjects + - improve tracking of PyObject to rpython object mapping + - support tp_as_{number, sequence, mapping, buffer} slots + + * CPyExt tweak: instead of "GIL not held when a CPython C extension module + calls PyXxx", we now silently acquire/release the GIL. Helps with + CPython C extension modules that call some PyXxx() functions without + holding the GIL (arguably, they are theorically buggy). + + * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. + It is a more flexible way to make RPython finalizers. Use this mechanism to + clean up handling of ``__del__`` methods, fixing issue #2287 + + * Generalize cpyext old-style buffers to more than just str/buffer, add + support for mmap + +* Bug Fixes + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* Numpy_: + + * Implement ufunc.outer on numpypy + +* Performance improvements: + + * Use bitstrings to compress lists of descriptors that are attached to an + EffectInfo + + * Remove most of the _ovf, _zer and _val operations from RPython. Kills + quite some code internally, and allows the JIT to do better + optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` + can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly + negative. + + +* Internal refactorings: + + * Reduce the size of generated C sources during translation by + refactoring function declarations + + * Remove a number of translation-time options that were not tested and + never used. Also fix a performance bug in the method cache + + * Reduce the size of generated code by using the same function objects in + all generated subclasses + + * Compile c snippets with -Werror, and fix warnings it exposed + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html +.. _Numpy: https://bitbucket.org/pypy/numpy + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + From pypy.commits at gmail.com Fri Jun 3 10:49:23 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:23 -0700 (PDT) Subject: [pypy-commit] pypy default: merge nonmovable-list which provides a way to get a raw pointer from a bytearray Message-ID: <575198f3.c8e51c0a.fee7.1f56@mx.google.com> Author: Matti Picus Branch: Changeset: r84906:f731f9ef48f8 Date: 2016-06-03 17:42 +0300 http://bitbucket.org/pypy/pypy/changeset/f731f9ef48f8/ Log: merge nonmovable-list which provides a way to get a raw pointer from a bytearray diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -184,7 +184,7 @@ def can_move(self, addr): return False - def malloc_fixedsize_nonmovable(self, typeid): + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): raise MemoryError def pin(self, addr): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -718,8 +718,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -628,8 +628,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -530,9 +530,10 @@ getfn(func, [SomeAddress()], annmodel.s_None) - self.malloc_nonmovable_ptr = getfn(GCClass.malloc_fixedsize_nonmovable, - [s_gc, s_typeid16], - s_gcref) + self.malloc_nonmovable_ptr = getfn( + GCClass.malloc_fixed_or_varsize_nonmovable, + [s_gc, s_typeid16, annmodel.SomeInteger()], + s_gcref) self.register_finalizer_ptr = getfn(GCClass.register_finalizer, [s_gc, @@ -773,12 +774,16 @@ c_has_light_finalizer = rmodel.inputconst(lltype.Bool, has_light_finalizer) + is_varsize = op.opname.endswith('_varsize') or flags.get('varsize') + if flags.get('nonmovable'): - assert op.opname == 'malloc' - assert not flags.get('varsize') + if not is_varsize: + v_length = rmodel.inputconst(lltype.Signed, 0) + else: + v_length = op.args[-1] malloc_ptr = self.malloc_nonmovable_ptr - args = [self.c_const_gc, c_type_id] - elif not op.opname.endswith('_varsize') and not flags.get('varsize'): + args = [self.c_const_gc, c_type_id, v_length] + elif not is_varsize: zero = flags.get('zero', False) if (self.malloc_fast_ptr is not None and not c_has_finalizer.value and diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1001,3 +1001,259 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.genop('gc_gettypeid', hop.args_v, resulttype=lltype.Signed) + +# ____________________________________________________________ + + +class _rawptr_missing_item(object): + pass +_rawptr_missing_item = _rawptr_missing_item() + + +class _ResizableListSupportingRawPtr(list): + """Calling this class is a no-op after translation. + + Before translation, it returns a new instance of + _ResizableListSupportingRawPtr, on which + rgc.nonmoving_raw_ptr_for_resizable_list() might be + used if needed. For now, only supports lists of chars. + """ + __slots__ = ('_raw_items',) # either None or a rffi.CCHARP + + def __init__(self, lst): + self._raw_items = None + self.__from_list(lst) + + def __resize(self): + """Called before an operation changes the size of the list""" + if self._raw_items is not None: + list.__init__(self, self.__as_list()) + self._raw_items = None + + def __from_list(self, lst): + """Initialize the list from a copy of the list 'lst'.""" + assert isinstance(lst, list) + for x in lst: + assert isinstance(x, str) and len(x) == 1 + if self is lst: + return + if len(self) != len(lst): + self.__resize() + if self._raw_items is None: + list.__init__(self, lst) + else: + assert len(self) == self._raw_items._obj.getlength() == len(lst) + for i in range(len(self)): + self._raw_items[i] = lst[i] + + def __as_list(self): + """Return a list (the same or a different one) which contains the + items in the regular way.""" + if self._raw_items is None: + return self + length = self._raw_items._obj.getlength() + assert length == len(self) + return [self._raw_items[i] for i in range(length)] + + def __getitem__(self, index): + if self._raw_items is None: + return list.__getitem__(self, index) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + return self._raw_items[index] + + def __setitem__(self, index, new): + if self._raw_items is None: + return list.__setitem__(self, index, new) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + self._raw_items[index] = new + + def __delitem__(self, index): + self.__resize() + list.__delitem__(self, index) + + def __getslice__(self, i, j): + return list.__getslice__(self.__as_list(), i, j) + + def __setslice__(self, i, j, new): + lst = self.__as_list() + list.__setslice__(lst, i, j, new) + self.__from_list(lst) + + def __delslice__(self, i, j): + lst = self.__as_list() + list.__delslice__(lst, i, j) + self.__from_list(lst) + + def __iter__(self): + try: + i = 0 + while True: + yield self[i] + i += 1 + except IndexError: + pass + + def __reversed__(self): + i = len(self) + while i > 0: + i -= 1 + yield self[i] + + def __contains__(self, item): + return list.__contains__(self.__as_list(), item) + + def __add__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(self.__as_list(), other) + + def __radd__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(other, self.__as_list()) + + def __iadd__(self, other): + self.__resize() + return list.__iadd__(self, other) + + def __eq__(self, other): + return list.__eq__(self.__as_list(), other) + def __ne__(self, other): + return list.__ne__(self.__as_list(), other) + def __ge__(self, other): + return list.__ge__(self.__as_list(), other) + def __gt__(self, other): + return list.__gt__(self.__as_list(), other) + def __le__(self, other): + return list.__le__(self.__as_list(), other) + def __lt__(self, other): + return list.__lt__(self.__as_list(), other) + + def __mul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __rmul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __imul__(self, other): + self.__resize() + return list.__imul__(self, other) + + def __repr__(self): + return '_ResizableListSupportingRawPtr(%s)' % ( + list.__repr__(self.__as_list()),) + + def append(self, object): + self.__resize() + return list.append(self, object) + + def count(self, value): + return list.count(self.__as_list(), value) + + def extend(self, iterable): + self.__resize() + return list.extend(self, iterable) + + def index(self, value, *start_stop): + return list.index(self.__as_list(), value, *start_stop) + + def insert(self, index, object): + self.__resize() + return list.insert(self, index, object) + + def pop(self, *opt_index): + self.__resize() + return list.pop(self, *opt_index) + + def remove(self, value): + self.__resize() + return list.remove(self, value) + + def reverse(self): + lst = self.__as_list() + list.reverse(lst) + self.__from_list(lst) + + def sort(self, *args, **kwds): + lst = self.__as_list() + list.sort(lst, *args, **kwds) + self.__from_list(lst) + + def _nonmoving_raw_ptr_for_resizable_list(self): + if self._raw_items is None: + existing_items = list(self) + from rpython.rtyper.lltypesystem import lltype, rffi + self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self), + flavor='raw', immortal=True) + self.__from_list(existing_items) + assert self._raw_items is not None + return self._raw_items + +def resizable_list_supporting_raw_ptr(lst): + return _ResizableListSupportingRawPtr(lst) + +def nonmoving_raw_ptr_for_resizable_list(lst): + assert isinstance(lst, _ResizableListSupportingRawPtr) + return lst._nonmoving_raw_ptr_for_resizable_list() + + +def _check_resizable_list_of_chars(s_list): + from rpython.annotator import model as annmodel + from rpython.rlib import debug + if annmodel.s_None.contains(s_list): + return # "None", will likely be generalized later + if not isinstance(s_list, annmodel.SomeList): + raise Exception("not a list, got %r" % (s_list,)) + if not isinstance(s_list.listdef.listitem.s_value, + (annmodel.SomeChar, annmodel.SomeImpossibleValue)): + raise debug.NotAListOfChars + s_list.listdef.resize() # must be resizable + +class Entry(ExtRegistryEntry): + _about_ = resizable_list_supporting_raw_ptr + + def compute_result_annotation(self, s_list): + _check_resizable_list_of_chars(s_list) + return s_list + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], 0) + +class Entry(ExtRegistryEntry): + _about_ = nonmoving_raw_ptr_for_resizable_list + + def compute_result_annotation(self, s_list): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.rtyper.llannotation import SomePtr + _check_resizable_list_of_chars(s_list) + return SomePtr(rffi.CCHARP) + + def specialize_call(self, hop): + v_list = hop.inputarg(hop.args_r[0], 0) + hop.exception_cannot_occur() # ignoring MemoryError + return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, + v_list) + +def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): + from rpython.rtyper.lltypesystem import lltype, rffi + array = ll_list.items + if can_move(array): + length = ll_list.length + new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length, + nonmovable=True) + i = 0 + while i < length: + new_array[i] = array[i] + i += 1 + ll_list.items = new_array + array = new_array + ptr = lltype.direct_arrayitems(array) + # ptr is a Ptr(FixedSizeArray(Char, 1)). Cast it to a rffi.CCHARP + return rffi.cast(rffi.CCHARP, ptr) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.test_llinterp import gengraph, interpret from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib import rgc # Force registration of gc.collect import gc import py, sys @@ -254,6 +254,153 @@ assert typer.custom_trace_funcs == [(TP, trace_func)] +def test_nonmoving_raw_ptr_for_resizable_list(): + def f(n): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst.append(chr(n)) + assert lst[3] == chr(n) + assert lst[-1] == chr(n) + # + ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst) + assert lst[:] == ['a', 'b', 'c', chr(n)] + assert lltype.typeOf(ptr) == rffi.CCHARP + assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)] + # + lst[-3] = 'X' + assert ptr[1] == 'X' + ptr[2] = 'Y' + assert lst[-2] == 'Y' + # + addr = rffi.cast(lltype.Signed, ptr) + ptr = rffi.cast(rffi.CCHARP, addr) + rgc.collect() # should not move lst.items + lst[-4] = 'g' + assert ptr[0] == 'g' + ptr[3] = 'H' + assert lst[-1] == 'H' + return lst + # + # direct untranslated run + lst = f(35) + assert isinstance(lst, rgc._ResizableListSupportingRawPtr) + # + # llinterp run + interpret(f, [35]) + # + # compilation with the GC transformer + import subprocess + from rpython.translator.interactive import Translation + # + def main(argv): + f(len(argv)) + print "OK!" + return 0 + # + t = Translation(main, gc="incminimark") + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') + +def test_ListSupportingRawPtr_direct(): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + + def check_nonresizing(): + assert lst[1] == lst[-2] == 'b' + lst[1] = 'X' + assert lst[1] == 'X' + lst[-1] = 'Y' + assert lst[1:3] == ['X', 'Y'] + assert lst[-2:9] == ['X', 'Y'] + lst[1:2] = 'B' + assert lst[:] == ['a', 'B', 'Y'] + assert list(iter(lst)) == ['a', 'B', 'Y'] + assert list(reversed(lst)) == ['Y', 'B', 'a'] + assert 'B' in lst + assert 'b' not in lst + assert p[0] == 'a' + assert p[1] == 'B' + assert p[2] == 'Y' + assert lst + ['*'] == ['a', 'B', 'Y', '*'] + assert ['*'] + lst == ['*', 'a', 'B', 'Y'] + assert lst + lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + base = ['8'] + base += lst + assert base == ['8', 'a', 'B', 'Y'] + assert lst == ['a', 'B', 'Y'] + assert ['a', 'B', 'Y'] == lst + assert ['a', 'B', 'Z'] != lst + assert ['a', 'B', 'Z'] > lst + assert ['a', 'B', 'Z'] >= lst + assert lst * 2 == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert 2 * lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert lst.count('B') == 1 + assert lst.index('Y') == 2 + lst.reverse() + assert lst == ['Y', 'B', 'a'] + lst.sort() + assert lst == ['B', 'Y', 'a'] + lst.sort(reverse=True) + assert lst == ['a', 'Y', 'B'] + lst[1] = 'b' + lst[2] = 'c' + assert list(lst) == ['a', 'b', 'c'] + + p = lst + check_nonresizing() + assert lst._raw_items is None + lst._nonmoving_raw_ptr_for_resizable_list() + p = lst._raw_items + check_nonresizing() + assert lst._raw_items == p + assert p[0] == 'a' + assert p[1] == 'b' + assert p[2] == 'c' + + def do_resizing_operation(): + del lst[1] + yield ['a', 'c'] + + lst[:2] = ['X'] + yield ['X', 'c'] + + del lst[:2] + yield ['c'] + + x = lst + x += ['t'] + yield ['a', 'b', 'c', 't'] + + x = lst + x *= 3 + yield ['a', 'b', 'c'] * 3 + + lst.append('f') + yield ['a', 'b', 'c', 'f'] + + lst.extend('fg') + yield ['a', 'b', 'c', 'f', 'g'] + + lst.insert(1, 'k') + yield ['a', 'k', 'b', 'c'] + + n = lst.pop(1) + assert n == 'b' + yield ['a', 'c'] + + lst.remove('c') + yield ['a', 'b'] + + assert lst == ['a', 'b', 'c'] + for expect in do_resizing_operation(): + assert lst == expect + assert lst._raw_items is None + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst._nonmoving_raw_ptr_for_resizable_list() # ____________________________________________________________ @@ -368,7 +515,6 @@ assert fq._triggered == 1 def test_finalizer_trigger_calls_too_much(self): - from rpython.rtyper.lltypesystem import lltype, rffi external_func = rffi.llexternal("foo", [], lltype.Void) # ^^^ with release_gil=True class X(object): diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -2174,7 +2174,8 @@ def malloc(T, n=None, flavor='gc', immortal=False, zero=False, - track_allocation=True, add_memory_pressure=False): + track_allocation=True, add_memory_pressure=False, + nonmovable=False): assert flavor in ('gc', 'raw') if zero or immortal: initialization = 'example' @@ -2200,7 +2201,8 @@ @analyzer_for(malloc) def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, - s_track_allocation=None, s_add_memory_pressure=None): + s_track_allocation=None, s_add_memory_pressure=None, + s_nonmovable=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2218,6 +2220,7 @@ assert s_track_allocation is None or s_track_allocation.is_constant() assert (s_add_memory_pressure is None or s_add_memory_pressure.is_constant()) + assert s_nonmovable is None or s_nonmovable.is_constant() # not sure how to call malloc() for the example 'p' in the # presence of s_extraargs r = SomePtr(Ptr(s_T.const)) diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -659,6 +659,7 @@ a[3] = 30 a[4] = 40 b0 = direct_arrayitems(a) + assert typeOf(b0) == Ptr(FixedSizeArray(Signed, 1)) b1 = direct_ptradd(b0, 1) b2 = direct_ptradd(b1, 1) b3 = direct_ptradd(b0, 3) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -348,7 +348,7 @@ @typer_for(lltype.malloc) def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, - i_add_memory_pressure=None): + i_add_memory_pressure=None, i_nonmovable=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' @@ -357,8 +357,10 @@ (i_flavor, lltype.Void), (i_zero, None), (i_track_allocation, None), - (i_add_memory_pressure, None)) - (v_flavor, v_zero, v_track_allocation, v_add_memory_pressure) = kwds_v + (i_add_memory_pressure, None), + (i_nonmovable, None)) + (v_flavor, v_zero, v_track_allocation, + v_add_memory_pressure, v_nonmovable) = kwds_v flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value @@ -368,6 +370,8 @@ flags['track_allocation'] = v_track_allocation.value if i_add_memory_pressure is not None: flags['add_memory_pressure'] = v_add_memory_pressure.value + if i_nonmovable is not None: + flags['nonmovable'] = v_nonmovable vlist.append(hop.inputconst(lltype.Void, flags)) assert 1 <= hop.nb_args <= 2 From pypy.commits at gmail.com Fri Jun 3 10:49:20 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:20 -0700 (PDT) Subject: [pypy-commit] pypy nonmovable-list: close branch to be merged Message-ID: <575198f0.692dc20a.8974c.3fde@mx.google.com> Author: Matti Picus Branch: nonmovable-list Changeset: r84904:2080acc8dead Date: 2016-06-03 17:39 +0300 http://bitbucket.org/pypy/pypy/changeset/2080acc8dead/ Log: close branch to be merged From pypy.commits at gmail.com Fri Jun 3 10:49:21 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:21 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-ext: close branch to be merged Message-ID: <575198f1.029a1c0a.671e2.79a7@mx.google.com> Author: Matti Picus Branch: cpyext-ext Changeset: r84905:bd0eeaa2eea3 Date: 2016-06-03 17:40 +0300 http://bitbucket.org/pypy/pypy/changeset/bd0eeaa2eea3/ Log: close branch to be merged From pypy.commits at gmail.com Fri Jun 3 10:49:25 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:25 -0700 (PDT) Subject: [pypy-commit] pypy default: merge cpyext-ext which extends cpyext compatibility Message-ID: <575198f5.512d1c0a.fa3c3.0fa3@mx.google.com> Author: Matti Picus Branch: Changeset: r84907:8f8354892934 Date: 2016-06-03 17:42 +0300 http://bitbucket.org/pypy/pypy/changeset/8f8354892934/ Log: merge cpyext-ext which extends cpyext compatibility diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -814,7 +814,7 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) << 1); /* check that '%s.%s' is " + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' @@ -991,7 +991,7 @@ prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') prnt(' int n = (%s) <= 0;' % (name,)) - prnt(' *o = (unsigned long long)((%s) << 0);' + prnt(' *o = (unsigned long long)((%s) | 0);' ' /* check that %s is an integer */' % (name, name)) if check_value is not None: if check_value > 0: @@ -1250,7 +1250,7 @@ def _emit_bytecode_UnknownIntegerType(self, tp, index): s = ('_cffi_prim_int(sizeof(%s), (\n' - ' ((%s)-1) << 0 /* check that %s is an integer type */\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' ' ) <= 0)' % (tp.name, tp.name, tp.name)) self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -243,6 +243,9 @@ def unicode_w(self, space): self._typed_unwrap_error(space, "unicode") + def bytearray_list_of_chars_w(self, space): + self._typed_unwrap_error(space, "bytearray") + def int_w(self, space, allow_conversion=True): # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w # raises w_TypeError diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -23,6 +23,7 @@ else: bases = [__base] self.bases = bases + # Used in cpyext to fill tp_as_buffer slots assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" self.buffer = __buffer self.heaptype = False diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -222,24 +222,43 @@ w_value.get_array_length() == length): # fast path: copying from exactly the correct type with w_value as source: - rffi.c_memcpy(target, source, ctitemsize * length) + source = rffi.cast(rffi.VOIDP, source) + target = rffi.cast(rffi.VOIDP, target) + size = rffi.cast(rffi.SIZE_T, ctitemsize * length) + rffi.c_memcpy(target, source, size) return # - # A fast path for [0:N] = "somestring". + # A fast path for [0:N] = "somestring" or some bytearray. from pypy.module._cffi_backend import ctypeprim space = self.space - if (space.isinstance_w(w_value, space.w_str) and - isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)): - from rpython.rtyper.annlowlevel import llstr - from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw - value = space.str_w(w_value) - if len(value) != length: - raise oefmt(space.w_ValueError, - "need a string of length %d, got %d", - length, len(value)) - copy_string_to_raw(llstr(value), target, 0, length) - return + if isinstance(ctitem, ctypeprim.W_CTypePrimitive) and ctitem.size == 1: + if space.isinstance_w(w_value, space.w_str): + from rpython.rtyper.annlowlevel import llstr + from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw + value = space.str_w(w_value) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a string of length %d, got %d", + length, len(value)) + copy_string_to_raw(llstr(value), target, 0, length) + return + if space.isinstance_w(w_value, space.w_bytearray): + value = w_value.bytearray_list_of_chars_w(space) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a bytearray of length %d, got %d", + length, len(value)) + self._copy_list_of_chars_to_raw(value, target, length) + return # + self._do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length) + + @staticmethod + def _do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length): + # general case, contains a loop + # (XXX is it worth adding a jitdriver here?) w_iter = space.iter(w_value) for i in range(length): try: @@ -260,6 +279,12 @@ raise oefmt(space.w_ValueError, "got more than %d values to unpack", length) + @staticmethod + def _copy_list_of_chars_to_raw(value, target, length): + # contains a loop, moved out of _do_setslice() + for i in range(length): + target[i] = value[i] + def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -43,9 +43,6 @@ else: return 'NULL' - def is_char_ptr_or_array(self): - return False - def is_unichar_ptr_or_array(self): return False diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -29,9 +29,6 @@ self.ctitem = ctitem self.can_cast_anything = could_cast_anything and ctitem.cast_anything - def is_char_ptr_or_array(self): - return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar) - def is_unichar_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar) @@ -54,12 +51,8 @@ value = rffi.cast(rffi.CCHARP, value) return cdataobj.W_CData(space, value, self) - def _convert_array_from_listview(self, cdata, w_ob): - if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path - return - # + def _convert_array_from_listview(self, cdata, lst_w): space = self.space - lst_w = space.listview(w_ob) if self.length >= 0 and len(lst_w) > self.length: raise oefmt(space.w_IndexError, "too many initializers for '%s' (got %d)", @@ -73,7 +66,10 @@ space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): - self._convert_array_from_listview(cdata, w_ob) + if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path + pass + else: + self._convert_array_from_listview(cdata, space.listview(w_ob)) elif (self.can_cast_anything or (self.ctitem.is_primitive_integer and self.ctitem.size == rffi.sizeof(lltype.Char))): diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -347,7 +347,7 @@ """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is -not meant to be used on the built-in types str, unicode, or bytearray +not meant to be used on the built-in types str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays.""" diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2686,10 +2686,19 @@ BCharArray = new_array_type( new_pointer_type(new_primitive_type("char")), None) py.test.raises(TypeError, newp, BCharArray, bytearray(b"foo")) - p = newp(BCharArray, 4) - buffer(p)[:] = bytearray(b"foo\x00") - assert len(p) == 4 - assert list(p) == [b"f", b"o", b"o", b"\x00"] + p = newp(BCharArray, 5) + buffer(p)[:] = bytearray(b"foo.\x00") + assert len(p) == 5 + assert list(p) == [b"f", b"o", b"o", b".", b"\x00"] + p[1:3] = bytearray(b"XY") + assert list(p) == [b"f", b"X", b"Y", b".", b"\x00"] + +def test_string_assignment_to_byte_array(): + BByteArray = new_array_type( + new_pointer_type(new_primitive_type("unsigned char")), None) + p = newp(BByteArray, 5) + p[0:3] = bytearray(b"XYZ") + assert list(p) == [ord("X"), ord("Y"), ord("Z"), 0, 0] # XXX hack if sys.version_info >= (3,): @@ -3317,13 +3326,12 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] -def test_from_buffer_not_str_unicode_bytearray(): +def test_from_buffer_not_str_unicode(): BChar = new_primitive_type("char") BCharP = new_pointer_type(BChar) BCharA = new_array_type(BCharP, None) py.test.raises(TypeError, from_buffer, BCharA, b"foo") py.test.raises(TypeError, from_buffer, BCharA, u+"foo") - py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) try: from __builtin__ import buffer except ImportError: @@ -3331,16 +3339,27 @@ else: py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - buffer(bytearray(b"foo"))) try: from __builtin__ import memoryview except ImportError: pass else: py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - memoryview(bytearray(b"foo"))) + +def test_from_buffer_bytearray(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + assert typeof(p) is BCharA + assert len(p) == 3 + assert repr(p) == "" + assert p[2] == b"z" + p[2] = b"." + assert a[2] == ord(".") + a[2] = ord("?") + assert p[2] == b"?" def test_from_buffer_more_cases(): try: diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -42,6 +42,7 @@ import pypy.module.cpyext.typeobject import pypy.module.cpyext.object import pypy.module.cpyext.bytesobject +import pypy.module.cpyext.bytearrayobject import pypy.module.cpyext.tupleobject import pypy.module.cpyext.setobject import pypy.module.cpyext.dictobject diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1429,7 +1429,7 @@ prefix=prefix) code = "#include \n" if use_micronumpy: - code += "#include /* api.py line 1290 */" + code += "#include /* api.py line 1290 */\n" code += "\n".join(functions) eci = build_eci(False, export_symbols, code, use_micronumpy) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/bytearrayobject.py @@ -0,0 +1,133 @@ +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.objectmodel import specialize, we_are_translated +from pypy.interpreter.error import OperationError, oefmt +from pypy.objspace.std.bytearrayobject import new_bytearray +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) +from pypy.module.cpyext.pyerrors import PyErr_BadArgument +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr, Py_IncRef) +# Type PyByteArrayObject represents a mutable array of bytes. +# The Python API is that of a sequence; +# the bytes are mapped to ints in [0, 256). +# Bytes are not characters; they may be used to encode characters. +# The only way to go between bytes and str/unicode is via encoding +# and decoding. +# For the convenience of C programmers, the bytes type is considered +# to contain a char pointer, not an unsigned char pointer. + +# Expose data as a rw cchar* only through PyByteArray_AsString +# Under this strategy the pointer could loose its synchronization with +# the underlying space.w_bytearray if PyByteArray_Resize is called, so +# hopefully the use of the pointer is short-lived + +PyByteArrayObjectStruct = lltype.ForwardReference() +PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) +PyByteArrayObjectFields = PyVarObjectFields +# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) + + at bootstrap_function +def init_bytearrayobject(space): + "Type description of PyByteArrayObject" + #make_typedescr(space.w_bytearray.layout.typedef, + # basestruct=PyByteArrayObject.TO, + # attach=bytearray_attach, + # dealloc=bytearray_dealloc, + # realize=bytearray_realize) + +PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") + +def bytearray_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyByteArrayObject with the given bytearray object + """ + py_ba = rffi.cast(PyByteArrayObject, py_obj) + py_ba.c_ob_size = len(space.str_w(w_obj)) + py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) + py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) + +def bytearray_realize(space, py_obj): + """ + Creates the bytearray in the interpreter. + """ + py_ba = rffi.cast(PyByteArrayObject, py_obj) + if not py_ba.c_ob_bytes: + py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, + flavor='raw', zero=True) + s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) + w_obj = space.wrap(s) + py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) + track_reference(space, py_obj, w_obj) + return w_obj + + at cpython_api([PyObject], lltype.Void, header=None) +def bytearray_dealloc(space, py_obj): + """Frees allocated PyByteArrayObject resources. + """ + py_ba = rffi.cast(PyByteArrayObject, py_obj) + if py_ba.c_ob_bytes: + lltype.free(py_ba.c_ob_bytes, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + + at cpython_api([PyObject], PyObject, result_is_ll=True) +def PyByteArray_FromObject(space, w_obj): + """Return a new bytearray object from any object, o, that implements the + buffer protocol. + + XXX expand about the buffer protocol, at least somewhere""" + w_buffer = space.call_function(space.w_bytearray, w_obj) + return make_ref(space, w_buffer) + + at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True) +def PyByteArray_FromStringAndSize(space, char_p, length): + """Create a new bytearray object from string and its length, len. On + failure, NULL is returned.""" + if char_p: + w_s = space.wrap(rffi.charpsize2str(char_p, length)) + else: + w_s = space.wrap(length) + w_buffer = space.call_function(space.w_bytearray, w_s) + return make_ref(space, w_buffer) + + at cpython_api([PyObject, PyObject], PyObject) +def PyByteArray_Concat(space, w_left, w_right): + """Concat bytearrays a and b and return a new bytearray with the result.""" + return space.add(w_left, w_right) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyByteArray_Size(space, w_obj): + """Return the size of bytearray after checking for a NULL pointer.""" + if not w_obj: + return 0 + return space.len_w(w_obj) + + at cpython_api([PyObject], rffi.CCHARP, error=0) +def PyByteArray_AsString(space, w_obj): + """Return the contents of bytearray as a char array after checking for a + NULL pointer.""" + if space.isinstance_w(w_obj, space.w_bytearray): + return w_obj.nonmovable_carray(space) + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) + + at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) +def PyByteArray_Resize(space, w_obj, newlen): + """Resize the internal buffer of bytearray to len.""" + if space.isinstance_w(w_obj, space.w_bytearray): + oldlen = space.len_w(w_obj) + if newlen > oldlen: + space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + elif oldlen > newlen: + assert newlen >= 0 + space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) + return 0 + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, build_type_checkers, - PyObjectFields, PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) + PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -106,6 +106,7 @@ #include "pythonrun.h" #include "pyerrors.h" #include "sysmodule.h" +#include "bytearrayobject.h" #include "stringobject.h" #include "descrobject.h" #include "tupleobject.h" diff --git a/pypy/module/cpyext/include/bytearrayobject.h b/pypy/module/cpyext/include/bytearrayobject.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/bytearrayobject.h @@ -0,0 +1,36 @@ +/* ByteArray object interface */ + +#ifndef Py_BYTEARRAYOBJECT_H +#define Py_BYTEARRAYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Type PyByteArrayObject represents a mutable array of bytes. + * The Python API is that of a sequence; + * the bytes are mapped to ints in [0, 256). + * Bytes are not characters; they may be used to encode characters. + * The only way to go between bytes and str/unicode is via encoding + * and decoding. + * While CPython exposes interfaces to this object, pypy does not + */ + +#define PyByteArray_GET_SIZE(op) PyByteArray_Size((PyObject*)(op)) +#define PyByteArray_AS_STRING(op) PyByteArray_AsString((PyObject*)(op)) + +/* Object layout */ +typedef struct { + PyObject_VAR_HEAD +#if 0 + int ob_exports; /* how many buffer exports */ + Py_ssize_t ob_alloc; /* How many bytes allocated */ + char *ob_bytes; +#endif +} PyByteArrayObject; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_BYTEARRAYOBJECT_H */ diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -59,63 +59,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyByteArray_Check(space, o): - """Return true if the object o is a bytearray object or an instance of a - subtype of the bytearray type.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyByteArray_CheckExact(space, o): - """Return true if the object o is a bytearray object, but not an instance of a - subtype of the bytearray type.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyByteArray_FromObject(space, o): - """Return a new bytearray object from any object, o, that implements the - buffer protocol. - - XXX expand about the buffer protocol, at least somewhere""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject) -def PyByteArray_FromStringAndSize(space, string, len): - """Create a new bytearray object from string and its length, len. On - failure, NULL is returned.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyByteArray_Concat(space, a, b): - """Concat bytearrays a and b and return a new bytearray with the result.""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_Size(space, bytearray): - """Return the size of bytearray after checking for a NULL pointer.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AsString(space, bytearray): - """Return the contents of bytearray as a char array after checking for a - NULL pointer.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) -def PyByteArray_Resize(space, bytearray, len): - """Resize the internal buffer of bytearray to len.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AS_STRING(space, bytearray): - """Macro version of PyByteArray_AsString().""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_GET_SIZE(space, bytearray): - """Macro version of PyByteArray_Size().""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCell_Check(space, ob): """Return true if ob is a cell object; ob must not be NULL.""" raise NotImplementedError diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -0,0 +1,199 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestStringObject(AppTestCpythonExtensionBase): + def test_basic(self): + module = self.import_extension('foo', [ + ("get_hello1", "METH_NOARGS", + """ + return PyByteArray_FromStringAndSize( + "Hello world", 11); + """), + ("get_hello2", "METH_NOARGS", + """ + return PyByteArray_FromStringAndSize("Hello world", 12); + """), + ("test_Size", "METH_NOARGS", + """ + PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); + int result = 0; + size_t expected_size; + + if(PyByteArray_Size(s) == 12) { + result = 1; + } + #ifdef PYPY_VERSION + expected_size = sizeof(void*)*3; + #elif defined Py_DEBUG + expected_size = 64; + #else + expected_size = 48; + #endif + if(s->ob_type->tp_basicsize != expected_size) + { + printf("tp_basicsize==%ld\\n", s->ob_type->tp_basicsize); + result = 0; + } + Py_DECREF(s); + return PyBool_FromLong(result); + """), + ("test_is_bytearray", "METH_VARARGS", + """ + return PyBool_FromLong(PyByteArray_Check(PyTuple_GetItem(args, 0))); + """)], prologue='#include ') + assert module.get_hello1() == 'Hello world' + assert module.get_hello2() == 'Hello world\x00' + assert module.test_Size() + assert module.test_is_bytearray(bytearray("")) + assert not module.test_is_bytearray(()) + + def test_bytearray_buffer_init(self): + module = self.import_extension('foo', [ + ("getbytearray", "METH_NOARGS", + """ + PyObject *s, *t; + char* c; + Py_ssize_t len; + + s = PyByteArray_FromStringAndSize(NULL, 4); + if (s == NULL) + return NULL; + t = PyByteArray_FromStringAndSize(NULL, 3); + if (t == NULL) + return NULL; + Py_DECREF(t); + c = PyByteArray_AsString(s); + if (c == NULL) + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } + c[0] = 'a'; + c[1] = 'b'; + c[2] = 0; + c[3] = 'c'; + return s; + """), + ]) + s = module.getbytearray() + assert len(s) == 4 + assert s == 'ab\x00c' + + def test_bytearray_mutable(self): + module = self.import_extension('foo', [ + ("mutable", "METH_NOARGS", + """ + PyObject *base; + char * p_str; + base = PyByteArray_FromStringAndSize("test", 10); + if (PyByteArray_GET_SIZE(base) != 10) + return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); + memcpy(PyByteArray_AS_STRING(base), "works", 6); + Py_INCREF(base); + return base; + """), + ]) + s = module.mutable() + if s == '\x00' * 10: + assert False, "no RW access to bytearray" + assert s[:6] == 'works\x00' + + def test_AsByteArray(self): + module = self.import_extension('foo', [ + ("getbytearray", "METH_NOARGS", + """ + PyObject* s1 = PyByteArray_FromStringAndSize("test", 4); + if (s1 == NULL) + return NULL; + char* c = PyByteArray_AsString(s1); + PyObject* s2 = PyByteArray_FromStringAndSize(c, 4); + Py_DECREF(s1); + return s2; + """), + ]) + s = module.getbytearray() + assert s == 'test' + + def test_manipulations(self): + module = self.import_extension('foo', [ + ("bytearray_from_string", "METH_VARARGS", + ''' + return PyByteArray_FromStringAndSize(PyString_AsString( + PyTuple_GetItem(args, 0)), 4); + ''' + ), + ("str_from_bytearray", "METH_VARARGS", + ''' + char * buf; + int n; + PyObject * obj; + obj = PyTuple_GetItem(args, 0); + buf = PyByteArray_AsString(obj); + if (buf == NULL) + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } + n = PyByteArray_Size(obj); + return PyString_FromStringAndSize(buf, n); + ''' + ), + ("concat", "METH_VARARGS", + """ + PyObject * ret, *right, *left; + PyObject *ba1, *ba2; + if (!PyArg_ParseTuple(args, "OO", &left, &right)) { + return PyString_FromString("parse failed"); + } + ba1 = PyByteArray_FromObject(left); + ba2 = PyByteArray_FromObject(right); + if (ba1 == NULL || ba2 == NULL) + { + /* exception should be set */ + return NULL; + } + ret = PyByteArray_Concat(ba1, ba2); + return ret; + """)]) + assert module.bytearray_from_string("huheduwe") == "huhe" + assert module.str_from_bytearray(bytearray('abc')) == 'abc' + raises(ValueError, module.str_from_bytearray, 4.0) + ret = module.concat('abc', 'def') + assert ret == 'abcdef' + assert not isinstance(ret, str) + assert isinstance(ret, bytearray) + raises(TypeError, module.concat, 'abc', u'def') + + def test_bytearray_resize(self): + module = self.import_extension('foo', [ + ("bytearray_resize", "METH_VARARGS", + ''' + PyObject *obj, *ba; + int newsize, oldsize, ret; + if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { + return PyString_FromString("parse failed"); + } + + ba = PyByteArray_FromObject(obj); + if (ba == NULL) + return NULL; + oldsize = PyByteArray_Size(ba); + if (oldsize == 0) + { + return PyString_FromString("oldsize is 0"); + } + ret = PyByteArray_Resize(ba, newsize); + if (ret != 0) + { + printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); + return NULL; + } + return ba; + ''' + )]) + ret = module.bytearray_resize('abc', 6) + assert len(ret) == 6,"%s, len=%d" % (ret, len(ret)) + assert ret == 'abc\x00\x00\x00' + ret = module.bytearray_resize('abcdefghi', 4) + assert len(ret) == 4,"%s, len=%d" % (ret, len(ret)) + assert ret == 'abcd' + diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -123,7 +123,6 @@ return result; ''') assert 'foo\0bar\0baz' == pybuffer('foo\0bar\0baz') - skip('PyByteArrayObject not implemented yet') assert 'foo\0bar\0baz' == pybuffer(bytearray('foo\0bar\0baz')) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr -import pytest import sys class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -124,7 +123,6 @@ obj = module.fooType.classmeth() assert obj is module.fooType - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython segfaults') def test_new(self): # XXX cpython segfaults but if run singly (with -k test_new) this passes module = self.import_module(name='foo') @@ -179,8 +177,6 @@ x = module.MetaType('name', (), {}) assert isinstance(x, type) assert isinstance(x, module.MetaType) - if self.runappdirect and '__pypy__' in sys.builtin_module_names: - skip('x is not callable when runappdirect??') x() def test_metaclass_compatible(self): @@ -190,17 +186,16 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.MetaType,), {}) assert isinstance(y, module.MetaType) - if self.runappdirect and '__pypy__' in sys.builtin_module_names: - skip('y is not callable when runappdirect??') x = y('something', (type(y),), {}) del x, y def test_metaclass_compatible2(self): - skip('type.__new__ does not check acceptable_as_base_class') + skip('fails even with -A, fooType has BASETYPE flag') # XXX FIX - must raise since fooType (which is a base type) # does not have flag Py_TPFLAGS_BASETYPE module = self.import_module(name='foo') raises(TypeError, module.MetaType, 'other', (module.fooType,), {}) + def test_sre(self): import sys for m in ['_sre', 'sre_compile', 'sre_constants', 'sre_parse', 're']: @@ -901,7 +896,6 @@ #print('calling module.footype()...') module.footype("X", (object,), {}) - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython fails') def test_app_subclass_of_c_type(self): # on cpython, the size changes (6 bytes added) module = self.import_module(name='foo') diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -671,7 +671,7 @@ @cpython_api([PyObject, PyObject], PyObject) def PyUnicode_Concat(space, w_left, w_right): """Concat two strings giving a new Unicode string.""" - return space.call_method(w_left, '__add__', w_right) + return space.add(w_left, w_right) @cpython_api([rffi.CWCHARP, rffi.CWCHARP, Py_ssize_t], lltype.Void) def Py_UNICODE_COPY(space, target, source, length): diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -5,6 +5,9 @@ from rpython.rlib.buffer import Buffer from rpython.rlib.rstring import StringBuilder, ByteListBuilder from rpython.rlib.debug import check_list_of_chars +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr, + nonmoving_raw_ptr_for_resizable_list) from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -24,7 +27,7 @@ def __init__(self, data): check_list_of_chars(data) - self.data = data + self.data = resizable_list_supporting_raw_ptr(data) def __repr__(self): """representation for debugging purposes""" @@ -42,6 +45,12 @@ def charbuf_w(self, space): return ''.join(self.data) + def bytearray_list_of_chars_w(self, space): + return self.data + + def nonmovable_carray(self, space): + return BytearrayBuffer(self.data, False).get_raw_address() + def _new(self, value): if value is self.data: value = value[:] @@ -223,11 +232,11 @@ else: if count < 0: raise oefmt(space.w_ValueError, "bytearray negative count") - self.data = ['\0'] * count + self.data = resizable_list_supporting_raw_ptr(['\0'] * count) return data = makebytearraydata_w(space, w_source) - self.data = data + self.data = resizable_list_supporting_raw_ptr(data) def descr_repr(self, space): s = self.data @@ -987,7 +996,7 @@ W_BytearrayObject.typedef = TypeDef( - "bytearray", + "bytearray", None, None, "read-write", __doc__ = BytearrayDocstrings.__doc__, __new__ = interp2app(W_BytearrayObject.descr_new), __hash__ = None, @@ -1246,6 +1255,9 @@ for i in range(len(string)): self.data[start + i] = string[i] + def get_raw_address(self): + return nonmoving_raw_ptr_for_resizable_list(self.data) + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py --- a/pypy/objspace/std/test/test_bufferobject.py +++ b/pypy/objspace/std/test/test_bufferobject.py @@ -201,5 +201,5 @@ def test_pypy_raw_address_base(self): raises(ValueError, buffer("foobar")._pypy_raw_address) raises(ValueError, buffer(u"foobar")._pypy_raw_address) - e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address) - assert 'BytearrayBuffer' in str(e.value) + a = buffer(bytearray("foobar"))._pypy_raw_address() + assert a != 0 diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -57,5 +57,5 @@ def test_pypy_raw_address_base(self): raises(ValueError, memoryview("foobar")._pypy_raw_address) - e = raises(ValueError, memoryview(bytearray("foobar"))._pypy_raw_address) - assert 'BytearrayBuffer' in str(e.value) + a = memoryview(bytearray("foobar"))._pypy_raw_address() + assert a != 0 From pypy.commits at gmail.com Fri Jun 3 10:49:27 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:49:27 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branches Message-ID: <575198f7.4e0b1c0a.33053.7222@mx.google.com> Author: Matti Picus Branch: Changeset: r84908:c6a47f55baba Date: 2016-06-03 17:48 +0300 http://bitbucket.org/pypy/pypy/changeset/c6a47f55baba/ Log: document merged branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -131,3 +131,15 @@ Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch at cpyext import time + +.. branch: nonmovable-list + +Add a way to ask "give me a raw pointer to this list's +items". Only for resizable lists of primitives. Turns the GcArray +nonmovable, possibly making a copy of it first. + +.. branch: cpyext-ext + +Finish the work already partially merged in cpyext-for-merge. Adds support +for ByteArrayObject using the nonmovable-list, which also enables +buffer(bytearray()) From pypy.commits at gmail.com Fri Jun 3 10:52:02 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:52:02 -0700 (PDT) Subject: [pypy-commit] pypy default: mark dead code Message-ID: <57519992.6513c20a.3014.33ea@mx.google.com> Author: Matti Picus Branch: Changeset: r84909:6f962c8d66e7 Date: 2016-06-03 17:51 +0300 http://bitbucket.org/pypy/pypy/changeset/6f962c8d66e7/ Log: mark dead code diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -40,40 +40,41 @@ PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -def bytearray_attach(space, py_obj, w_obj): - """ - Fills a newly allocated PyByteArrayObject with the given bytearray object - """ - py_ba = rffi.cast(PyByteArrayObject, py_obj) - py_ba.c_ob_size = len(space.str_w(w_obj)) - py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) - py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) +# XXX dead code to be removed +#def bytearray_attach(space, py_obj, w_obj): +# """ +# Fills a newly allocated PyByteArrayObject with the given bytearray object +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# py_ba.c_ob_size = len(space.str_w(w_obj)) +# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) +# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -def bytearray_realize(space, py_obj): - """ - Creates the bytearray in the interpreter. - """ - py_ba = rffi.cast(PyByteArrayObject, py_obj) - if not py_ba.c_ob_bytes: - py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, - flavor='raw', zero=True) - s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) - w_obj = space.wrap(s) - py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - track_reference(space, py_obj, w_obj) - return w_obj +#def bytearray_realize(space, py_obj): +# """ +# Creates the bytearray in the interpreter. +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# if not py_ba.c_ob_bytes: +# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, +# flavor='raw', zero=True) +# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) +# w_obj = space.wrap(s) +# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) +# track_reference(space, py_obj, w_obj) +# return w_obj - at cpython_api([PyObject], lltype.Void, header=None) -def bytearray_dealloc(space, py_obj): - """Frees allocated PyByteArrayObject resources. - """ - py_ba = rffi.cast(PyByteArrayObject, py_obj) - if py_ba.c_ob_bytes: - lltype.free(py_ba.c_ob_bytes, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) - -#_______________________________________________________________________ +#@cpython_api([PyObject], lltype.Void, header=None) +#def bytearray_dealloc(space, py_obj): +# """Frees allocated PyByteArrayObject resources. +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# if py_ba.c_ob_bytes: +# lltype.free(py_ba.c_ob_bytes, flavor="raw") +# from pypy.module.cpyext.object import PyObject_dealloc +# PyObject_dealloc(space, py_obj) +# +_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) def PyByteArray_FromObject(space, w_obj): From pypy.commits at gmail.com Fri Jun 3 10:55:01 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 03 Jun 2016 07:55:01 -0700 (PDT) Subject: [pypy-commit] pypy default: typo Message-ID: <57519a45.4f961c0a.50cf9.7ae1@mx.google.com> Author: Matti Picus Branch: Changeset: r84910:85ffcf013c4f Date: 2016-06-03 17:54 +0300 http://bitbucket.org/pypy/pypy/changeset/85ffcf013c4f/ Log: typo diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -73,8 +73,8 @@ # lltype.free(py_ba.c_ob_bytes, flavor="raw") # from pypy.module.cpyext.object import PyObject_dealloc # PyObject_dealloc(space, py_obj) -# -_______________________________________________________________________ + +#_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) def PyByteArray_FromObject(space, w_obj): From pypy.commits at gmail.com Fri Jun 3 14:56:19 2016 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Jun 2016 11:56:19 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default (f731f9ef48f8) Message-ID: <5751d2d3.6513c20a.3014.ffff8bda@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r84911:b56f881df46b Date: 2016-06-03 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/b56f881df46b/ Log: hg merge default (f731f9ef48f8) diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -0,0 +1,139 @@ +============ +PyPy2.7 v5.3 +============ + +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. +This release includes further improvements for the CAPI compatibility layer +which we call cpyext. In addtion to complete support for lxml, we now pass +most (more than 90%) of the upstream numpy test suite, and much of SciPy is +supported as well. + +We also improved the speed of ... and ... + +We updated cffi_ to ... + +You can download the PyPy2.7 v5.3 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy +.. _cffi: https://cffi.readthedocs.org +.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html +.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.1 released in April 2016) +========================================================= + +* New features: + + * Merge a major expansion of the C-API support in cpyext, here are some of + the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible + - fix many edge cases exposed by fixing tests to run with -A + - issequence() logic matches cpython + - make PyStringObject and PyUnicodeObject field names compatible with cpython + - add prelminary support for PyDateTime_* + - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, + PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, + - PyAnySet_CheckExact, PyUnicode_Concat + - improve support for PyGILState_Ensure, PyGILState_Release, and thread + primitives, also find a case where CPython will allow thread creation + before PyEval_InitThreads is run, dissallow on PyPy + - create a PyObject-specific list strategy + - rewrite slot assignment for typeobjects + - improve tracking of PyObject to rpython object mapping + - support tp_as_{number, sequence, mapping, buffer} slots + + * CPyExt tweak: instead of "GIL not held when a CPython C extension module + calls PyXxx", we now silently acquire/release the GIL. Helps with + CPython C extension modules that call some PyXxx() functions without + holding the GIL (arguably, they are theorically buggy). + + * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. + It is a more flexible way to make RPython finalizers. Use this mechanism to + clean up handling of ``__del__`` methods, fixing issue #2287 + + * Generalize cpyext old-style buffers to more than just str/buffer, add + support for mmap + +* Bug Fixes + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* Numpy_: + + * Implement ufunc.outer on numpypy + +* Performance improvements: + + * Use bitstrings to compress lists of descriptors that are attached to an + EffectInfo + + * Remove most of the _ovf, _zer and _val operations from RPython. Kills + quite some code internally, and allows the JIT to do better + optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` + can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly + negative. + + +* Internal refactorings: + + * Reduce the size of generated C sources during translation by + refactoring function declarations + + * Remove a number of translation-time options that were not tested and + never used. Also fix a performance bug in the method cache + + * Reduce the size of generated code by using the same function objects in + all generated subclasses + + * Compile c snippets with -Werror, and fix warnings it exposed + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html +.. _Numpy: https://bitbucket.org/pypy/numpy + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -669,7 +669,6 @@ self.emit_jump(ops.JUMP_FORWARD, end) self.use_next_block(next_except) self.emit_op(ops.END_FINALLY) # this END_FINALLY will always re-raise - self.is_dead_code() self.use_next_block(otherwise) self.visit_sequence(tr.orelse) self.use_next_block(end) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "3.3.5" /* PyPy version as a string */ -#define PYPY_VERSION "5.2.0-alpha0" -#define PYPY_VERSION_NUM 0x05020000 +#define PYPY_VERSION "5.3.0-alpha0" +#define PYPY_VERSION_NUM 0x05030000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 2, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -460,9 +460,8 @@ def _hexstring_to_array(space, s): data = [] length = len(s) - i = -2 + i = 0 while True: - i += 2 while i < length and s[i] == ' ': i += 1 if i >= length: @@ -483,6 +482,7 @@ "non-hexadecimal number found in fromhex() arg at " "position %d", i + 1) data.append(chr(top*16 + bot)) + i += 2 return data diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -184,7 +184,7 @@ def can_move(self, addr): return False - def malloc_fixedsize_nonmovable(self, typeid): + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): raise MemoryError def pin(self, addr): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -718,8 +718,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -628,8 +628,9 @@ return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0, alloc_young=True) + def malloc_fixed_or_varsize_nonmovable(self, typeid, length): + # length==0 for fixedsize + obj = self.external_malloc(typeid, length, alloc_young=True) return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -530,9 +530,10 @@ getfn(func, [SomeAddress()], annmodel.s_None) - self.malloc_nonmovable_ptr = getfn(GCClass.malloc_fixedsize_nonmovable, - [s_gc, s_typeid16], - s_gcref) + self.malloc_nonmovable_ptr = getfn( + GCClass.malloc_fixed_or_varsize_nonmovable, + [s_gc, s_typeid16, annmodel.SomeInteger()], + s_gcref) self.register_finalizer_ptr = getfn(GCClass.register_finalizer, [s_gc, @@ -773,12 +774,16 @@ c_has_light_finalizer = rmodel.inputconst(lltype.Bool, has_light_finalizer) + is_varsize = op.opname.endswith('_varsize') or flags.get('varsize') + if flags.get('nonmovable'): - assert op.opname == 'malloc' - assert not flags.get('varsize') + if not is_varsize: + v_length = rmodel.inputconst(lltype.Signed, 0) + else: + v_length = op.args[-1] malloc_ptr = self.malloc_nonmovable_ptr - args = [self.c_const_gc, c_type_id] - elif not op.opname.endswith('_varsize') and not flags.get('varsize'): + args = [self.c_const_gc, c_type_id, v_length] + elif not is_varsize: zero = flags.get('zero', False) if (self.malloc_fast_ptr is not None and not c_has_finalizer.value and diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1001,3 +1001,259 @@ def specialize_call(self, hop): hop.exception_cannot_occur() return hop.genop('gc_gettypeid', hop.args_v, resulttype=lltype.Signed) + +# ____________________________________________________________ + + +class _rawptr_missing_item(object): + pass +_rawptr_missing_item = _rawptr_missing_item() + + +class _ResizableListSupportingRawPtr(list): + """Calling this class is a no-op after translation. + + Before translation, it returns a new instance of + _ResizableListSupportingRawPtr, on which + rgc.nonmoving_raw_ptr_for_resizable_list() might be + used if needed. For now, only supports lists of chars. + """ + __slots__ = ('_raw_items',) # either None or a rffi.CCHARP + + def __init__(self, lst): + self._raw_items = None + self.__from_list(lst) + + def __resize(self): + """Called before an operation changes the size of the list""" + if self._raw_items is not None: + list.__init__(self, self.__as_list()) + self._raw_items = None + + def __from_list(self, lst): + """Initialize the list from a copy of the list 'lst'.""" + assert isinstance(lst, list) + for x in lst: + assert isinstance(x, str) and len(x) == 1 + if self is lst: + return + if len(self) != len(lst): + self.__resize() + if self._raw_items is None: + list.__init__(self, lst) + else: + assert len(self) == self._raw_items._obj.getlength() == len(lst) + for i in range(len(self)): + self._raw_items[i] = lst[i] + + def __as_list(self): + """Return a list (the same or a different one) which contains the + items in the regular way.""" + if self._raw_items is None: + return self + length = self._raw_items._obj.getlength() + assert length == len(self) + return [self._raw_items[i] for i in range(length)] + + def __getitem__(self, index): + if self._raw_items is None: + return list.__getitem__(self, index) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + return self._raw_items[index] + + def __setitem__(self, index, new): + if self._raw_items is None: + return list.__setitem__(self, index, new) + if index < 0: + index += len(self) + if not (0 <= index < len(self)): + raise IndexError + self._raw_items[index] = new + + def __delitem__(self, index): + self.__resize() + list.__delitem__(self, index) + + def __getslice__(self, i, j): + return list.__getslice__(self.__as_list(), i, j) + + def __setslice__(self, i, j, new): + lst = self.__as_list() + list.__setslice__(lst, i, j, new) + self.__from_list(lst) + + def __delslice__(self, i, j): + lst = self.__as_list() + list.__delslice__(lst, i, j) + self.__from_list(lst) + + def __iter__(self): + try: + i = 0 + while True: + yield self[i] + i += 1 + except IndexError: + pass + + def __reversed__(self): + i = len(self) + while i > 0: + i -= 1 + yield self[i] + + def __contains__(self, item): + return list.__contains__(self.__as_list(), item) + + def __add__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(self.__as_list(), other) + + def __radd__(self, other): + if isinstance(other, _ResizableListSupportingRawPtr): + other = other.__as_list() + return list.__add__(other, self.__as_list()) + + def __iadd__(self, other): + self.__resize() + return list.__iadd__(self, other) + + def __eq__(self, other): + return list.__eq__(self.__as_list(), other) + def __ne__(self, other): + return list.__ne__(self.__as_list(), other) + def __ge__(self, other): + return list.__ge__(self.__as_list(), other) + def __gt__(self, other): + return list.__gt__(self.__as_list(), other) + def __le__(self, other): + return list.__le__(self.__as_list(), other) + def __lt__(self, other): + return list.__lt__(self.__as_list(), other) + + def __mul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __rmul__(self, other): + return list.__mul__(self.__as_list(), other) + + def __imul__(self, other): + self.__resize() + return list.__imul__(self, other) + + def __repr__(self): + return '_ResizableListSupportingRawPtr(%s)' % ( + list.__repr__(self.__as_list()),) + + def append(self, object): + self.__resize() + return list.append(self, object) + + def count(self, value): + return list.count(self.__as_list(), value) + + def extend(self, iterable): + self.__resize() + return list.extend(self, iterable) + + def index(self, value, *start_stop): + return list.index(self.__as_list(), value, *start_stop) + + def insert(self, index, object): + self.__resize() + return list.insert(self, index, object) + + def pop(self, *opt_index): + self.__resize() + return list.pop(self, *opt_index) + + def remove(self, value): + self.__resize() + return list.remove(self, value) + + def reverse(self): + lst = self.__as_list() + list.reverse(lst) + self.__from_list(lst) + + def sort(self, *args, **kwds): + lst = self.__as_list() + list.sort(lst, *args, **kwds) + self.__from_list(lst) + + def _nonmoving_raw_ptr_for_resizable_list(self): + if self._raw_items is None: + existing_items = list(self) + from rpython.rtyper.lltypesystem import lltype, rffi + self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self), + flavor='raw', immortal=True) + self.__from_list(existing_items) + assert self._raw_items is not None + return self._raw_items + +def resizable_list_supporting_raw_ptr(lst): + return _ResizableListSupportingRawPtr(lst) + +def nonmoving_raw_ptr_for_resizable_list(lst): + assert isinstance(lst, _ResizableListSupportingRawPtr) + return lst._nonmoving_raw_ptr_for_resizable_list() + + +def _check_resizable_list_of_chars(s_list): + from rpython.annotator import model as annmodel + from rpython.rlib import debug + if annmodel.s_None.contains(s_list): + return # "None", will likely be generalized later + if not isinstance(s_list, annmodel.SomeList): + raise Exception("not a list, got %r" % (s_list,)) + if not isinstance(s_list.listdef.listitem.s_value, + (annmodel.SomeChar, annmodel.SomeImpossibleValue)): + raise debug.NotAListOfChars + s_list.listdef.resize() # must be resizable + +class Entry(ExtRegistryEntry): + _about_ = resizable_list_supporting_raw_ptr + + def compute_result_annotation(self, s_list): + _check_resizable_list_of_chars(s_list) + return s_list + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], 0) + +class Entry(ExtRegistryEntry): + _about_ = nonmoving_raw_ptr_for_resizable_list + + def compute_result_annotation(self, s_list): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.rtyper.llannotation import SomePtr + _check_resizable_list_of_chars(s_list) + return SomePtr(rffi.CCHARP) + + def specialize_call(self, hop): + v_list = hop.inputarg(hop.args_r[0], 0) + hop.exception_cannot_occur() # ignoring MemoryError + return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, + v_list) + +def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): + from rpython.rtyper.lltypesystem import lltype, rffi + array = ll_list.items + if can_move(array): + length = ll_list.length + new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length, + nonmovable=True) + i = 0 + while i < length: + new_array[i] = array[i] + i += 1 + ll_list.items = new_array + array = new_array + ptr = lltype.direct_arrayitems(array) + # ptr is a Ptr(FixedSizeArray(Char, 1)). Cast it to a rffi.CCHARP + return rffi.cast(rffi.CCHARP, ptr) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.test_llinterp import gengraph, interpret from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib import rgc # Force registration of gc.collect import gc import py, sys @@ -254,6 +254,153 @@ assert typer.custom_trace_funcs == [(TP, trace_func)] +def test_nonmoving_raw_ptr_for_resizable_list(): + def f(n): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst.append(chr(n)) + assert lst[3] == chr(n) + assert lst[-1] == chr(n) + # + ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst) + assert lst[:] == ['a', 'b', 'c', chr(n)] + assert lltype.typeOf(ptr) == rffi.CCHARP + assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)] + # + lst[-3] = 'X' + assert ptr[1] == 'X' + ptr[2] = 'Y' + assert lst[-2] == 'Y' + # + addr = rffi.cast(lltype.Signed, ptr) + ptr = rffi.cast(rffi.CCHARP, addr) + rgc.collect() # should not move lst.items + lst[-4] = 'g' + assert ptr[0] == 'g' + ptr[3] = 'H' + assert lst[-1] == 'H' + return lst + # + # direct untranslated run + lst = f(35) + assert isinstance(lst, rgc._ResizableListSupportingRawPtr) + # + # llinterp run + interpret(f, [35]) + # + # compilation with the GC transformer + import subprocess + from rpython.translator.interactive import Translation + # + def main(argv): + f(len(argv)) + print "OK!" + return 0 + # + t = Translation(main, gc="incminimark") + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') + +def test_ListSupportingRawPtr_direct(): + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + + def check_nonresizing(): + assert lst[1] == lst[-2] == 'b' + lst[1] = 'X' + assert lst[1] == 'X' + lst[-1] = 'Y' + assert lst[1:3] == ['X', 'Y'] + assert lst[-2:9] == ['X', 'Y'] + lst[1:2] = 'B' + assert lst[:] == ['a', 'B', 'Y'] + assert list(iter(lst)) == ['a', 'B', 'Y'] + assert list(reversed(lst)) == ['Y', 'B', 'a'] + assert 'B' in lst + assert 'b' not in lst + assert p[0] == 'a' + assert p[1] == 'B' + assert p[2] == 'Y' + assert lst + ['*'] == ['a', 'B', 'Y', '*'] + assert ['*'] + lst == ['*', 'a', 'B', 'Y'] + assert lst + lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + base = ['8'] + base += lst + assert base == ['8', 'a', 'B', 'Y'] + assert lst == ['a', 'B', 'Y'] + assert ['a', 'B', 'Y'] == lst + assert ['a', 'B', 'Z'] != lst + assert ['a', 'B', 'Z'] > lst + assert ['a', 'B', 'Z'] >= lst + assert lst * 2 == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert 2 * lst == ['a', 'B', 'Y', 'a', 'B', 'Y'] + assert lst.count('B') == 1 + assert lst.index('Y') == 2 + lst.reverse() + assert lst == ['Y', 'B', 'a'] + lst.sort() + assert lst == ['B', 'Y', 'a'] + lst.sort(reverse=True) + assert lst == ['a', 'Y', 'B'] + lst[1] = 'b' + lst[2] = 'c' + assert list(lst) == ['a', 'b', 'c'] + + p = lst + check_nonresizing() + assert lst._raw_items is None + lst._nonmoving_raw_ptr_for_resizable_list() + p = lst._raw_items + check_nonresizing() + assert lst._raw_items == p + assert p[0] == 'a' + assert p[1] == 'b' + assert p[2] == 'c' + + def do_resizing_operation(): + del lst[1] + yield ['a', 'c'] + + lst[:2] = ['X'] + yield ['X', 'c'] + + del lst[:2] + yield ['c'] + + x = lst + x += ['t'] + yield ['a', 'b', 'c', 't'] + + x = lst + x *= 3 + yield ['a', 'b', 'c'] * 3 + + lst.append('f') + yield ['a', 'b', 'c', 'f'] + + lst.extend('fg') + yield ['a', 'b', 'c', 'f', 'g'] + + lst.insert(1, 'k') + yield ['a', 'k', 'b', 'c'] + + n = lst.pop(1) + assert n == 'b' + yield ['a', 'c'] + + lst.remove('c') + yield ['a', 'b'] + + assert lst == ['a', 'b', 'c'] + for expect in do_resizing_operation(): + assert lst == expect + assert lst._raw_items is None + lst = ['a', 'b', 'c'] + lst = rgc.resizable_list_supporting_raw_ptr(lst) + lst._nonmoving_raw_ptr_for_resizable_list() # ____________________________________________________________ @@ -368,7 +515,6 @@ assert fq._triggered == 1 def test_finalizer_trigger_calls_too_much(self): - from rpython.rtyper.lltypesystem import lltype, rffi external_func = rffi.llexternal("foo", [], lltype.Void) # ^^^ with release_gil=True class X(object): diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -2174,7 +2174,8 @@ def malloc(T, n=None, flavor='gc', immortal=False, zero=False, - track_allocation=True, add_memory_pressure=False): + track_allocation=True, add_memory_pressure=False, + nonmovable=False): assert flavor in ('gc', 'raw') if zero or immortal: initialization = 'example' @@ -2200,7 +2201,8 @@ @analyzer_for(malloc) def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, - s_track_allocation=None, s_add_memory_pressure=None): + s_track_allocation=None, s_add_memory_pressure=None, + s_nonmovable=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2218,6 +2220,7 @@ assert s_track_allocation is None or s_track_allocation.is_constant() assert (s_add_memory_pressure is None or s_add_memory_pressure.is_constant()) + assert s_nonmovable is None or s_nonmovable.is_constant() # not sure how to call malloc() for the example 'p' in the # presence of s_extraargs r = SomePtr(Ptr(s_T.const)) diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -659,6 +659,7 @@ a[3] = 30 a[4] = 40 b0 = direct_arrayitems(a) + assert typeOf(b0) == Ptr(FixedSizeArray(Signed, 1)) b1 = direct_ptradd(b0, 1) b2 = direct_ptradd(b1, 1) b3 = direct_ptradd(b0, 3) diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -348,7 +348,7 @@ @typer_for(lltype.malloc) def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, - i_add_memory_pressure=None): + i_add_memory_pressure=None, i_nonmovable=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' @@ -357,8 +357,10 @@ (i_flavor, lltype.Void), (i_zero, None), (i_track_allocation, None), - (i_add_memory_pressure, None)) - (v_flavor, v_zero, v_track_allocation, v_add_memory_pressure) = kwds_v + (i_add_memory_pressure, None), + (i_nonmovable, None)) + (v_flavor, v_zero, v_track_allocation, + v_add_memory_pressure, v_nonmovable) = kwds_v flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value @@ -368,6 +370,8 @@ flags['track_allocation'] = v_track_allocation.value if i_add_memory_pressure is not None: flags['add_memory_pressure'] = v_add_memory_pressure.value + if i_nonmovable is not None: + flags['nonmovable'] = v_nonmovable vlist.append(hop.inputconst(lltype.Void, flags)) assert 1 <= hop.nb_args <= 2 From pypy.commits at gmail.com Fri Jun 3 15:03:11 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 03 Jun 2016 12:03:11 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement pyopcode BUILD_SET_UNPACK Message-ID: <5751d46f.08371c0a.7877d.5541@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84912:282bcd4967a7 Date: 2016-06-03 21:02 +0200 http://bitbucket.org/pypy/pypy/changeset/282bcd4967a7/ Log: Implement pyopcode BUILD_SET_UNPACK diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1326,11 +1326,12 @@ def BUILD_SET_UNPACK(self, itemcount, next_instr): w_sum = self.space.newset() - for i in range(itemcount): - self.space.updateset() #implement? - w_item = self.popvalue() - self.space.call_method(w_set, 'add', w_item) - self.pushvalue(w_set) + for i in range(itemcount, 0, -1): + self.space.call_method(w_sum, 'update', self.space.peek(i)) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_sum) def BUILD_TUPLE_UNPACK(self, itemcount, next_instr): w_sum = self.space.newset() From pypy.commits at gmail.com Fri Jun 3 15:29:46 2016 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Jun 2016 12:29:46 -0700 (PDT) Subject: [pypy-commit] pypy default: sync W_BytearrayObject.descr_fromhex() with py3k Message-ID: <5751daaa.e778c20a.56641.ffff95ad@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84913:6480742316c7 Date: 2016-06-03 20:29 +0100 http://bitbucket.org/pypy/pypy/changeset/6480742316c7/ Log: sync W_BytearrayObject.descr_fromhex() with py3k diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -19,7 +19,6 @@ from pypy.objspace.std.bytesobject import W_BytesObject from pypy.objspace.std.util import get_positive_index -NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d" class W_BytearrayObject(W_Root): @@ -176,27 +175,7 @@ @staticmethod def descr_fromhex(space, w_bytearraytype, w_hexstring): hexstring = space.str_w(w_hexstring) - hexstring = hexstring.lower() - data = [] - length = len(hexstring) - i = 0 - while True: - while i < length and hexstring[i] == ' ': - i += 1 - if i >= length: - break - if i + 1 == length: - raise oefmt(space.w_ValueError, NON_HEX_MSG, i) - - top = _hex_digit_to_int(hexstring[i]) - if top == -1: - raise oefmt(space.w_ValueError, NON_HEX_MSG, i) - bot = _hex_digit_to_int(hexstring[i+1]) - if bot == -1: - raise oefmt(space.w_ValueError, NON_HEX_MSG, i + 1) - data.append(chr(top*16 + bot)) - i += 2 - + data = _hexstring_to_array(space, hexstring) # in CPython bytearray.fromhex is a staticmethod, so # we ignore w_type and always return a bytearray return new_bytearray(space, space.w_bytearray, data) @@ -571,6 +550,31 @@ return val - 87 return -1 +NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d" + +def _hexstring_to_array(space, s): + s = s.lower() + data = [] + length = len(s) + i = 0 + while True: + while i < length and s[i] == ' ': + i += 1 + if i >= length: + break + if i + 1 == length: + raise oefmt(space.w_ValueError, NON_HEX_MSG, i) + + top = _hex_digit_to_int(s[i]) + if top == -1: + raise oefmt(space.w_ValueError, NON_HEX_MSG, i) + bot = _hex_digit_to_int(s[i + 1]) + if bot == -1: + raise oefmt(space.w_ValueError, NON_HEX_MSG, i + 1) + data.append(chr(top * 16 + bot)) + i += 2 + return data + class BytearrayDocstrings: """bytearray(iterable_of_ints) -> bytearray @@ -1257,7 +1261,7 @@ def get_raw_address(self): return nonmoving_raw_ptr_for_resizable_list(self.data) - + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): From pypy.commits at gmail.com Fri Jun 3 16:36:03 2016 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Jun 2016 13:36:03 -0700 (PDT) Subject: [pypy-commit] pypy default: More py3k sync Message-ID: <5751ea33.c6e41c0a.778d1.75db@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84914:16c557706539 Date: 2016-06-03 21:35 +0100 http://bitbucket.org/pypy/pypy/changeset/16c557706539/ Log: More py3k sync diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -180,22 +180,12 @@ # we ignore w_type and always return a bytearray return new_bytearray(space, space.w_bytearray, data) - def descr_init(self, space, __args__): - # this is on the silly side - w_source, w_encoding, w_errors = __args__.parse_obj( - None, 'bytearray', init_signature, init_defaults) - + @unwrap_spec(encoding='str_or_None', errors='str_or_None') + def descr_init(self, space, w_source=None, encoding=None, errors=None): if w_source is None: w_source = space.wrap('') - - # Unicode argument - if w_encoding is not None: - from pypy.objspace.std.unicodeobject import ( - _get_encoding_and_errors, encode_object - ) - encoding, errors = _get_encoding_and_errors(space, w_encoding, - w_errors) - + if encoding is not None: + from pypy.objspace.std.unicodeobject import encode_object # if w_source is an integer this correctly raises a # TypeError the CPython error message is: "encoding or # errors without a string argument" ours is: "expected @@ -546,6 +536,8 @@ val = ord(d) if 47 < val < 58: return val - 48 + if 64 < val < 71: + return val - 55 if 96 < val < 103: return val - 87 return -1 @@ -553,7 +545,6 @@ NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d" def _hexstring_to_array(space, s): - s = s.lower() data = [] length = len(s) i = 0 @@ -1144,9 +1135,6 @@ ) W_BytearrayObject.typedef.flag_sequence_bug_compat = True -init_signature = Signature(['source', 'encoding', 'errors'], None, None) -init_defaults = [None, None, None] - # XXX share the code again with the stuff in listobject.py def _delitem_slice_helper(space, items, start, step, slicelength): From pypy.commits at gmail.com Fri Jun 3 17:03:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Fri, 03 Jun 2016 14:03:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Change kind to func in codegen _make_call Message-ID: <5751f0bc.41811c0a.d577.ffff82a2@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84915:dcb2c9c112db Date: 2016-06-03 23:03 +0200 http://bitbucket.org/pypy/pypy/changeset/dcb2c9c112db/ Log: Change kind to func in codegen _make_call diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1113,7 +1113,7 @@ nkw = 0 nseen = 0 # the number of positional arguments on the stack for elt in args: - if isinstance(elt.kind, ast.Starred): + if isinstance(elt.func, ast.Starred): # A star-arg. If we've seen positional arguments, # pack the positional arguments into a tuple. if nseen != 0: From pypy.commits at gmail.com Fri Jun 3 21:58:47 2016 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Jun 2016 18:58:47 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <575235d7.012dc20a.8121c.06f4@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r84916:ef5a7f148b27 Date: 2016-06-04 02:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ef5a7f148b27/ Log: hg merge default Port cpyext changes to bytearray diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -814,7 +814,7 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) << 1); /* check that '%s.%s' is " + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' @@ -991,7 +991,7 @@ prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') prnt(' int n = (%s) <= 0;' % (name,)) - prnt(' *o = (unsigned long long)((%s) << 0);' + prnt(' *o = (unsigned long long)((%s) | 0);' ' /* check that %s is an integer */' % (name, name)) if check_value is not None: if check_value > 0: @@ -1250,7 +1250,7 @@ def _emit_bytecode_UnknownIntegerType(self, tp, index): s = ('_cffi_prim_int(sizeof(%s), (\n' - ' ((%s)-1) << 0 /* check that %s is an integer type */\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' ' ) <= 0)' % (tp.name, tp.name, tp.name)) self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -131,3 +131,15 @@ Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch at cpyext import time + +.. branch: nonmovable-list + +Add a way to ask "give me a raw pointer to this list's +items". Only for resizable lists of primitives. Turns the GcArray +nonmovable, possibly making a copy of it first. + +.. branch: cpyext-ext + +Finish the work already partially merged in cpyext-for-merge. Adds support +for ByteArrayObject using the nonmovable-list, which also enables +buffer(bytearray()) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -251,6 +251,9 @@ def identifier_w(self, space): self._typed_unwrap_error(space, "string") + def bytearray_list_of_chars_w(self, space): + self._typed_unwrap_error(space, "bytearray") + def int_w(self, space, allow_conversion=True): # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w # raises w_TypeError diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -23,6 +23,7 @@ else: bases = [__base] self.bases = bases + # Used in cpyext to fill tp_as_buffer slots assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" self.buffer = __buffer self.heaptype = False diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -222,24 +222,43 @@ w_value.get_array_length() == length): # fast path: copying from exactly the correct type with w_value as source: - rffi.c_memcpy(target, source, ctitemsize * length) + source = rffi.cast(rffi.VOIDP, source) + target = rffi.cast(rffi.VOIDP, target) + size = rffi.cast(rffi.SIZE_T, ctitemsize * length) + rffi.c_memcpy(target, source, size) return # - # A fast path for [0:N] = "somestring". + # A fast path for [0:N] = "somestring" or some bytearray. from pypy.module._cffi_backend import ctypeprim space = self.space - if (space.isinstance_w(w_value, space.w_str) and - isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)): - from rpython.rtyper.annlowlevel import llstr - from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw - value = space.str_w(w_value) - if len(value) != length: - raise oefmt(space.w_ValueError, - "need a string of length %d, got %d", - length, len(value)) - copy_string_to_raw(llstr(value), target, 0, length) - return + if isinstance(ctitem, ctypeprim.W_CTypePrimitive) and ctitem.size == 1: + if space.isinstance_w(w_value, space.w_str): + from rpython.rtyper.annlowlevel import llstr + from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw + value = space.str_w(w_value) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a string of length %d, got %d", + length, len(value)) + copy_string_to_raw(llstr(value), target, 0, length) + return + if space.isinstance_w(w_value, space.w_bytearray): + value = w_value.bytearray_list_of_chars_w(space) + if len(value) != length: + raise oefmt(space.w_ValueError, + "need a bytearray of length %d, got %d", + length, len(value)) + self._copy_list_of_chars_to_raw(value, target, length) + return # + self._do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length) + + @staticmethod + def _do_setslice_iterate(space, ctitem, w_value, target, ctitemsize, + length): + # general case, contains a loop + # (XXX is it worth adding a jitdriver here?) w_iter = space.iter(w_value) for i in range(length): try: @@ -260,6 +279,12 @@ raise oefmt(space.w_ValueError, "got more than %d values to unpack", length) + @staticmethod + def _copy_list_of_chars_to_raw(value, target, length): + # contains a loop, moved out of _do_setslice() + for i in range(length): + target[i] = value[i] + def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -43,9 +43,6 @@ else: return 'NULL' - def is_char_ptr_or_array(self): - return False - def is_unichar_ptr_or_array(self): return False diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -31,9 +31,6 @@ self.ctitem = ctitem self.can_cast_anything = could_cast_anything and ctitem.cast_anything - def is_char_ptr_or_array(self): - return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar) - def is_unichar_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar) @@ -56,12 +53,8 @@ value = rffi.cast(rffi.CCHARP, value) return cdataobj.W_CData(space, value, self) - def _convert_array_from_listview(self, cdata, w_ob): - if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path - return - # + def _convert_array_from_listview(self, cdata, lst_w): space = self.space - lst_w = space.listview(w_ob) if self.length >= 0 and len(lst_w) > self.length: raise oefmt(space.w_IndexError, "too many initializers for '%s' (got %d)", @@ -75,7 +68,10 @@ space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): - self._convert_array_from_listview(cdata, w_ob) + if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path + pass + else: + self._convert_array_from_listview(cdata, space.listview(w_ob)) elif (self.can_cast_anything or (self.ctitem.is_primitive_integer and self.ctitem.size == rffi.sizeof(lltype.Char))): diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -347,7 +347,7 @@ """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is -not meant to be used on the built-in types str, unicode, or bytearray +not meant to be used on the built-in types str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays.""" diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2686,10 +2686,19 @@ BCharArray = new_array_type( new_pointer_type(new_primitive_type("char")), None) py.test.raises(TypeError, newp, BCharArray, bytearray(b"foo")) - p = newp(BCharArray, 4) - buffer(p)[:] = bytearray(b"foo\x00") - assert len(p) == 4 - assert list(p) == [b"f", b"o", b"o", b"\x00"] + p = newp(BCharArray, 5) + buffer(p)[:] = bytearray(b"foo.\x00") + assert len(p) == 5 + assert list(p) == [b"f", b"o", b"o", b".", b"\x00"] + p[1:3] = bytearray(b"XY") + assert list(p) == [b"f", b"X", b"Y", b".", b"\x00"] + +def test_string_assignment_to_byte_array(): + BByteArray = new_array_type( + new_pointer_type(new_primitive_type("unsigned char")), None) + p = newp(BByteArray, 5) + p[0:3] = bytearray(b"XYZ") + assert list(p) == [ord("X"), ord("Y"), ord("Z"), 0, 0] # XXX hack if sys.version_info >= (3,): @@ -3317,13 +3326,12 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] -def test_from_buffer_not_str_unicode_bytearray(): +def test_from_buffer_not_str_unicode(): BChar = new_primitive_type("char") BCharP = new_pointer_type(BChar) BCharA = new_array_type(BCharP, None) py.test.raises(TypeError, from_buffer, BCharA, b"foo") py.test.raises(TypeError, from_buffer, BCharA, u+"foo") - py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) try: from __builtin__ import buffer except ImportError: @@ -3331,16 +3339,27 @@ else: py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - buffer(bytearray(b"foo"))) try: from __builtin__ import memoryview except ImportError: pass else: py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, - memoryview(bytearray(b"foo"))) + +def test_from_buffer_bytearray(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + assert typeof(p) is BCharA + assert len(p) == 3 + assert repr(p) == "" + assert p[2] == b"z" + p[2] = b"." + assert a[2] == ord(".") + a[2] = ord("?") + assert p[2] == b"?" def test_from_buffer_more_cases(): try: diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -42,6 +42,7 @@ import pypy.module.cpyext.typeobject import pypy.module.cpyext.object import pypy.module.cpyext.bytesobject +import pypy.module.cpyext.bytearrayobject import pypy.module.cpyext.tupleobject import pypy.module.cpyext.setobject import pypy.module.cpyext.dictobject diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1433,7 +1433,7 @@ prefix=prefix) code = "#include \n" if use_micronumpy: - code += "#include /* api.py line 1290 */" + code += "#include /* api.py line 1290 */\n" code += "\n".join(functions) eci = build_eci(False, export_symbols, code, use_micronumpy) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/bytearrayobject.py @@ -0,0 +1,134 @@ +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.objectmodel import specialize, we_are_translated +from pypy.interpreter.error import OperationError, oefmt +from pypy.objspace.std.bytearrayobject import new_bytearray +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) +from pypy.module.cpyext.pyerrors import PyErr_BadArgument +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr, Py_IncRef) +# Type PyByteArrayObject represents a mutable array of bytes. +# The Python API is that of a sequence; +# the bytes are mapped to ints in [0, 256). +# Bytes are not characters; they may be used to encode characters. +# The only way to go between bytes and str/unicode is via encoding +# and decoding. +# For the convenience of C programmers, the bytes type is considered +# to contain a char pointer, not an unsigned char pointer. + +# Expose data as a rw cchar* only through PyByteArray_AsString +# Under this strategy the pointer could loose its synchronization with +# the underlying space.w_bytearray if PyByteArray_Resize is called, so +# hopefully the use of the pointer is short-lived + +PyByteArrayObjectStruct = lltype.ForwardReference() +PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) +PyByteArrayObjectFields = PyVarObjectFields +# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) +cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) + + at bootstrap_function +def init_bytearrayobject(space): + "Type description of PyByteArrayObject" + #make_typedescr(space.w_bytearray.layout.typedef, + # basestruct=PyByteArrayObject.TO, + # attach=bytearray_attach, + # dealloc=bytearray_dealloc, + # realize=bytearray_realize) + +PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") + +# XXX dead code to be removed +#def bytearray_attach(space, py_obj, w_obj): +# """ +# Fills a newly allocated PyByteArrayObject with the given bytearray object +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# py_ba.c_ob_size = len(space.str_w(w_obj)) +# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) +# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) + +#def bytearray_realize(space, py_obj): +# """ +# Creates the bytearray in the interpreter. +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# if not py_ba.c_ob_bytes: +# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, +# flavor='raw', zero=True) +# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) +# w_obj = space.wrap(s) +# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) +# track_reference(space, py_obj, w_obj) +# return w_obj + +#@cpython_api([PyObject], lltype.Void, header=None) +#def bytearray_dealloc(space, py_obj): +# """Frees allocated PyByteArrayObject resources. +# """ +# py_ba = rffi.cast(PyByteArrayObject, py_obj) +# if py_ba.c_ob_bytes: +# lltype.free(py_ba.c_ob_bytes, flavor="raw") +# from pypy.module.cpyext.object import PyObject_dealloc +# PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + + at cpython_api([PyObject], PyObject, result_is_ll=True) +def PyByteArray_FromObject(space, w_obj): + """Return a new bytearray object from any object, o, that implements the + buffer protocol. + + XXX expand about the buffer protocol, at least somewhere""" + w_buffer = space.call_function(space.w_bytearray, w_obj) + return make_ref(space, w_buffer) + + at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True) +def PyByteArray_FromStringAndSize(space, char_p, length): + """Create a new bytearray object from string and its length, len. On + failure, NULL is returned.""" + if char_p: + w_s = space.wrapbytes(rffi.charpsize2str(char_p, length)) + else: + w_s = space.newint(length) + w_buffer = space.call_function(space.w_bytearray, w_s) + return make_ref(space, w_buffer) + + at cpython_api([PyObject, PyObject], PyObject) +def PyByteArray_Concat(space, w_left, w_right): + """Concat bytearrays a and b and return a new bytearray with the result.""" + return space.add(w_left, w_right) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyByteArray_Size(space, w_obj): + """Return the size of bytearray after checking for a NULL pointer.""" + if not w_obj: + return 0 + return space.len_w(w_obj) + + at cpython_api([PyObject], rffi.CCHARP, error=0) +def PyByteArray_AsString(space, w_obj): + """Return the contents of bytearray as a char array after checking for a + NULL pointer.""" + if space.isinstance_w(w_obj, space.w_bytearray): + return w_obj.nonmovable_carray(space) + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) + + at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) +def PyByteArray_Resize(space, w_obj, newlen): + """Resize the internal buffer of bytearray to len.""" + if space.isinstance_w(w_obj, space.w_bytearray): + oldlen = space.len_w(w_obj) + if newlen > oldlen: + space.call_method(w_obj, 'extend', space.wrapbytes('\x00' * (newlen - oldlen))) + elif oldlen > newlen: + assert newlen >= 0 + space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) + return 0 + else: + raise oefmt(space.w_TypeError, + "expected bytearray object, %T found", w_obj) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, build_type_checkers, - PyObjectFields, PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) + PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -114,6 +114,7 @@ #include "pythonrun.h" #include "pyerrors.h" #include "sysmodule.h" +#include "bytearrayobject.h" #include "descrobject.h" #include "tupleobject.h" #include "dictobject.h" diff --git a/pypy/module/cpyext/include/bytearrayobject.h b/pypy/module/cpyext/include/bytearrayobject.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/bytearrayobject.h @@ -0,0 +1,36 @@ +/* ByteArray object interface */ + +#ifndef Py_BYTEARRAYOBJECT_H +#define Py_BYTEARRAYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Type PyByteArrayObject represents a mutable array of bytes. + * The Python API is that of a sequence; + * the bytes are mapped to ints in [0, 256). + * Bytes are not characters; they may be used to encode characters. + * The only way to go between bytes and str/unicode is via encoding + * and decoding. + * While CPython exposes interfaces to this object, pypy does not + */ + +#define PyByteArray_GET_SIZE(op) PyByteArray_Size((PyObject*)(op)) +#define PyByteArray_AS_STRING(op) PyByteArray_AsString((PyObject*)(op)) + +/* Object layout */ +typedef struct { + PyObject_VAR_HEAD +#if 0 + int ob_exports; /* how many buffer exports */ + Py_ssize_t ob_alloc; /* How many bytes allocated */ + char *ob_bytes; +#endif +} PyByteArrayObject; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_BYTEARRAYOBJECT_H */ diff --git a/pypy/module/cpyext/include/bytesobject.h b/pypy/module/cpyext/include/bytesobject.h --- a/pypy/module/cpyext/include/bytesobject.h +++ b/pypy/module/cpyext/include/bytesobject.h @@ -61,9 +61,6 @@ #define PyString_CHECK_INTERNED(op) (((PyStringObject *)(op))->ob_sstate) -#define PyByteArray_Check(obj) \ - PyObject_IsInstance(obj, (PyObject *)&PyByteArray_Type) - #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -41,57 +41,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyByteArray_CheckExact(space, o): - """Return true if the object o is a bytearray object, but not an instance of a - subtype of the bytearray type.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyByteArray_FromObject(space, o): - """Return a new bytearray object from any object, o, that implements the - buffer protocol. - - XXX expand about the buffer protocol, at least somewhere""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject) -def PyByteArray_FromStringAndSize(space, string, len): - """Create a new bytearray object from string and its length, len. On - failure, NULL is returned.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyByteArray_Concat(space, a, b): - """Concat bytearrays a and b and return a new bytearray with the result.""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_Size(space, bytearray): - """Return the size of bytearray after checking for a NULL pointer.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AsString(space, bytearray): - """Return the contents of bytearray as a char array after checking for a - NULL pointer.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) -def PyByteArray_Resize(space, bytearray, len): - """Resize the internal buffer of bytearray to len.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) -def PyByteArray_AS_STRING(space, bytearray): - """Macro version of PyByteArray_AsString().""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyByteArray_GET_SIZE(space, bytearray): - """Macro version of PyByteArray_Size().""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCell_Check(space, ob): """Return true if ob is a cell object; ob must not be NULL.""" raise NotImplementedError diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -0,0 +1,198 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestStringObject(AppTestCpythonExtensionBase): + def test_basic(self): + module = self.import_extension('foo', [ + ("get_hello1", "METH_NOARGS", + """ + return PyByteArray_FromStringAndSize( + "Hello world", 11); + """), + ("get_hello2", "METH_NOARGS", + """ + return PyByteArray_FromStringAndSize("Hello world", 12); + """), + ("test_Size", "METH_NOARGS", + """ + PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); + int result = 0; + size_t expected_size; + + if(PyByteArray_Size(s) == 12) { + result = 1; + } + #ifdef PYPY_VERSION + expected_size = sizeof(void*)*3; + #elif defined Py_DEBUG + expected_size = 64; + #else + expected_size = 48; + #endif + if(s->ob_type->tp_basicsize != expected_size) + { + printf("tp_basicsize==%ld\\n", s->ob_type->tp_basicsize); + result = 0; + } + Py_DECREF(s); + return PyBool_FromLong(result); + """), + ("test_is_bytearray", "METH_VARARGS", + """ + return PyBool_FromLong(PyByteArray_Check(PyTuple_GetItem(args, 0))); + """)], prologue='#include ') + assert module.get_hello1() == b'Hello world' + assert module.get_hello2() == b'Hello world\x00' + assert module.test_Size() + assert module.test_is_bytearray(bytearray(b"")) + assert not module.test_is_bytearray(()) + + def test_bytearray_buffer_init(self): + module = self.import_extension('foo', [ + ("getbytearray", "METH_NOARGS", + """ + PyObject *s, *t; + char* c; + Py_ssize_t len; + + s = PyByteArray_FromStringAndSize(NULL, 4); + if (s == NULL) + return NULL; + t = PyByteArray_FromStringAndSize(NULL, 3); + if (t == NULL) + return NULL; + Py_DECREF(t); + c = PyByteArray_AsString(s); + if (c == NULL) + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } + c[0] = 'a'; + c[1] = 'b'; + c[2] = 0; + c[3] = 'c'; + return s; + """), + ]) + s = module.getbytearray() + assert len(s) == 4 + assert s == b'ab\x00c' + + def test_bytearray_mutable(self): + module = self.import_extension('foo', [ + ("mutable", "METH_NOARGS", + """ + PyObject *base; + char * p_str; + base = PyByteArray_FromStringAndSize("test", 10); + if (PyByteArray_GET_SIZE(base) != 10) + return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); + memcpy(PyByteArray_AS_STRING(base), "works", 6); + Py_INCREF(base); + return base; + """), + ]) + s = module.mutable() + if s == b'\x00' * 10: + assert False, "no RW access to bytearray" + assert s[:6] == b'works\x00' + + def test_AsByteArray(self): + module = self.import_extension('foo', [ + ("getbytearray", "METH_NOARGS", + """ + PyObject* s1 = PyByteArray_FromStringAndSize("test", 4); + if (s1 == NULL) + return NULL; + char* c = PyByteArray_AsString(s1); + PyObject* s2 = PyByteArray_FromStringAndSize(c, 4); + Py_DECREF(s1); + return s2; + """), + ]) + s = module.getbytearray() + assert s == b'test' + + def test_manipulations(self): + module = self.import_extension('foo', [ + ("bytearray_from_bytes", "METH_VARARGS", + ''' + return PyByteArray_FromStringAndSize(PyBytes_AsString( + PyTuple_GetItem(args, 0)), 4); + ''' + ), + ("bytes_from_bytearray", "METH_VARARGS", + ''' + char * buf; + int n; + PyObject * obj; + obj = PyTuple_GetItem(args, 0); + buf = PyByteArray_AsString(obj); + if (buf == NULL) + { + PyErr_SetString(PyExc_ValueError, "non-null bytearray object expected"); + return NULL; + } + n = PyByteArray_Size(obj); + return PyBytes_FromStringAndSize(buf, n); + ''' + ), + ("concat", "METH_VARARGS", + """ + PyObject * ret, *right, *left; + PyObject *ba1, *ba2; + if (!PyArg_ParseTuple(args, "OO", &left, &right)) { + return PyUnicode_FromString("parse failed"); + } + ba1 = PyByteArray_FromObject(left); + ba2 = PyByteArray_FromObject(right); + if (ba1 == NULL || ba2 == NULL) + { + /* exception should be set */ + return NULL; + } + ret = PyByteArray_Concat(ba1, ba2); + return ret; + """)]) + assert module.bytearray_from_bytes(b"huheduwe") == b"huhe" + assert module.bytes_from_bytearray(bytearray(b'abc')) == b'abc' + raises(ValueError, module.bytes_from_bytearray, 4.0) + ret = module.concat(b'abc', b'def') + assert ret == b'abcdef' + assert not isinstance(ret, str) + assert isinstance(ret, bytearray) + raises(TypeError, module.concat, b'abc', u'def') + + def test_bytearray_resize(self): + module = self.import_extension('foo', [ + ("bytearray_resize", "METH_VARARGS", + ''' + PyObject *obj, *ba; + int newsize, oldsize, ret; + if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { + return PyUnicode_FromString("parse failed"); + } + + ba = PyByteArray_FromObject(obj); + if (ba == NULL) + return NULL; + oldsize = PyByteArray_Size(ba); + if (oldsize == 0) + { + return PyUnicode_FromString("oldsize is 0"); + } + ret = PyByteArray_Resize(ba, newsize); + if (ret != 0) + { + printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); + return NULL; + } + return ba; + ''' + )]) + ret = module.bytearray_resize(b'abc', 6) + assert len(ret) == 6,"%s, len=%d" % (ret, len(ret)) + assert ret == b'abc\x00\x00\x00' + ret = module.bytearray_resize(b'abcdefghi', 4) + assert len(ret) == 4,"%s, len=%d" % (ret, len(ret)) + assert ret == b'abcd' diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -123,7 +123,8 @@ return result; ''') assert b'foo\0bar\0baz' == pybuffer(b'foo\0bar\0baz') - + return # XXX + assert b'foo\0bar\0baz' == pybuffer(bytearray(b'foo\0bar\0baz')) def test_pyarg_parse_string_fails(self): """ diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,8 +5,8 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr +import sys import pytest -import sys class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): @@ -124,7 +124,6 @@ obj = module.fooType.classmeth() assert obj is module.fooType - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython segfaults') def test_new(self): # XXX cpython segfaults but if run singly (with -k test_new) this passes module = self.import_module(name='foo') @@ -179,8 +178,6 @@ x = module.MetaType('name', (), {}) assert isinstance(x, type) assert isinstance(x, module.MetaType) - if self.runappdirect and '__pypy__' in sys.builtin_module_names: - skip('x is not callable when runappdirect??') x() def test_metaclass_compatible(self): @@ -190,13 +187,11 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.MetaType,), {}) assert isinstance(y, module.MetaType) - if self.runappdirect and '__pypy__' in sys.builtin_module_names: - skip('y is not callable when runappdirect??') x = y('something', (type(y),), {}) del x, y def test_metaclass_compatible2(self): - skip('type.__new__ does not check acceptable_as_base_class') + skip('fails even with -A, fooType has BASETYPE flag') # XXX FIX - must raise since fooType (which is a base type) # does not have flag Py_TPFLAGS_BASETYPE module = self.import_module(name='foo') @@ -880,7 +875,6 @@ #print('calling module.footype()...') module.footype("X", (object,), {}) - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='cpython fails') def test_app_subclass_of_c_type(self): # on cpython, the size changes (6 bytes added) module = self.import_module(name='foo') diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -792,7 +792,7 @@ @cpython_api([PyObject, PyObject], PyObject) def PyUnicode_Concat(space, w_left, w_right): """Concat two strings giving a new Unicode string.""" - return space.call_method(w_left, '__add__', w_right) + return space.add(w_left, w_right) @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=CANNOT_FAIL) def PyUnicode_CompareWithASCIIString(space, w_uni, string): diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -5,6 +5,9 @@ from rpython.rlib.buffer import Buffer from rpython.rlib.rstring import StringBuilder, ByteListBuilder from rpython.rlib.debug import check_list_of_chars +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr, + nonmoving_raw_ptr_for_resizable_list) from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -25,7 +28,7 @@ def __init__(self, data): check_list_of_chars(data) - self.data = data + self.data = resizable_list_supporting_raw_ptr(data) def __repr__(self): """representation for debugging purposes""" @@ -34,6 +37,12 @@ def buffer_w(self, space, flags): return BytearrayBuffer(self.data, False) + def bytearray_list_of_chars_w(self, space): + return self.data + + def nonmovable_carray(self, space): + return BytearrayBuffer(self.data, False).get_raw_address() + def _new(self, value): if value is self.data: value = value[:] @@ -176,7 +185,8 @@ @unwrap_spec(encoding='str_or_None', errors='str_or_None') def descr_init(self, space, w_source=None, encoding=None, errors=None): assert isinstance(self, W_BytearrayObject) - self.data = newbytesdata_w(space, w_source, encoding, errors) + data = newbytesdata_w(space, w_source, encoding, errors) + self.data = resizable_list_supporting_raw_ptr(data) def descr_repr(self, space): s = self.data @@ -457,6 +467,8 @@ return val - 87 return -1 +NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d" + def _hexstring_to_array(space, s): data = [] length = len(s) @@ -467,21 +479,15 @@ if i >= length: break if i + 1 == length: - raise oefmt(space.w_ValueError, - "non-hexadecimal number found in fromhex() arg at " - "position %d", i) + raise oefmt(space.w_ValueError, NON_HEX_MSG, i) top = _hex_digit_to_int(s[i]) if top == -1: - raise oefmt(space.w_ValueError, - "non-hexadecimal number found in fromhex() arg at " - "position %d", i) - bot = _hex_digit_to_int(s[i+1]) + raise oefmt(space.w_ValueError, NON_HEX_MSG, i) + bot = _hex_digit_to_int(s[i + 1]) if bot == -1: - raise oefmt(space.w_ValueError, - "non-hexadecimal number found in fromhex() arg at " - "position %d", i + 1) - data.append(chr(top*16 + bot)) + raise oefmt(space.w_ValueError, NON_HEX_MSG, i + 1) + data.append(chr(top * 16 + bot)) i += 2 return data @@ -1184,6 +1190,9 @@ for i in range(len(string)): self.data[start + i] = string[i] + def get_raw_address(self): + return nonmoving_raw_ptr_for_resizable_list(self.data) + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -159,5 +159,5 @@ def test_pypy_raw_address_base(self): raises(ValueError, memoryview(b"foobar")._pypy_raw_address) - e = raises(ValueError, memoryview(bytearray(b"foobar"))._pypy_raw_address) - assert 'BytearrayBuffer' in str(e.value) + a = memoryview(bytearray(b"foobar"))._pypy_raw_address() + assert a != 0 From pypy.commits at gmail.com Sat Jun 4 11:30:48 2016 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Jun 2016 08:30:48 -0700 (PDT) Subject: [pypy-commit] pypy default: Move pypydir definition from conftest.py to pypy/__init__.py Message-ID: <5752f428.078e1c0a.d6fc3.ffffc144@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84917:6174f94686c9 Date: 2016-06-04 16:30 +0100 http://bitbucket.org/pypy/pypy/changeset/6174f94686c9/ Log: Move pypydir definition from conftest.py to pypy/__init__.py diff --git a/pypy/__init__.py b/pypy/__init__.py --- a/pypy/__init__.py +++ b/pypy/__init__.py @@ -1,4 +1,5 @@ -# Empty +import os +pypydir = os.path.realpath(os.path.dirname(__file__)) # XXX Should be empty again, soon. # XXX hack for win64: diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -1,4 +1,4 @@ -import py, pytest, sys, os, textwrap +import py, pytest, sys, textwrap from inspect import isclass # pytest settings @@ -10,8 +10,6 @@ # option = None -pypydir = os.path.realpath(os.path.dirname(__file__)) - def braindead_deindent(self): """monkeypatch that wont end up doing stupid in the python tokenizer""" text = '\n'.join(self.lines) diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -9,7 +9,7 @@ from rpython.config.config import to_optparse, make_dict, SUPPRESS_USAGE from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace -from pypy.conftest import pypydir +from pypy import pypydir from rpython.rlib import rthread from pypy.module.thread import os_thread @@ -293,7 +293,7 @@ self.hack_for_cffi_modules(driver) return self.get_entry_point(config) - + def hack_for_cffi_modules(self, driver): # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports @@ -320,7 +320,7 @@ while not basedir.join('include').exists(): _basedir = basedir.dirpath() if _basedir == basedir: - raise ValueError('interpreter %s not inside pypy repo', + raise ValueError('interpreter %s not inside pypy repo', str(exename)) basedir = _basedir modules = self.config.objspace.usemodules.getpaths() diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -6,7 +6,7 @@ import sys, os, re, runpy, subprocess from rpython.tool.udir import udir from contextlib import contextmanager -from pypy.conftest import pypydir +from pypy import pypydir from lib_pypy._pypy_interact import irc_header try: @@ -291,7 +291,7 @@ child.expect('>>>') # banner if irc_topic: assert irc_header in child.before - else: + else: assert irc_header not in child.before def test_help(self): @@ -1074,4 +1074,4 @@ # assert it did not crash finally: sys.path[:] = old_sys_path - + diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -1,4 +1,4 @@ -from pypy.conftest import pypydir +from pypy import pypydir from rpython.tool.udir import udir import py import sys @@ -70,7 +70,7 @@ f.write(source) child = self.spawn(['--withmod-_minimal_curses', str(f)]) child.expect('ok!') - + class TestCCurses(object): """ Test compiled version """ diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -4,7 +4,7 @@ import py -from pypy.conftest import pypydir +from pypy import pypydir from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import ll2ctypes diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -4,7 +4,7 @@ import py, pytest -from pypy.conftest import pypydir +from pypy import pypydir from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import lltype, ll2ctypes from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -5,7 +5,7 @@ from pypy.objspace.std import StdObjSpace from rpython.tool.udir import udir from pypy.tool.pytest.objspace import gettestobjspace -from pypy.conftest import pypydir +from pypy import pypydir from rpython.translator.c.test.test_extfunc import need_sparse_files from rpython.rlib import rposix import os diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -2,7 +2,7 @@ Implementation of interpreter-level 'sys' routines. """ import os -import pypy +from pypy import pypydir # ____________________________________________________________ # @@ -20,7 +20,6 @@ def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path # Initialize the default path - pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) srcdir = os.path.dirname(pypydir) path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -14,7 +14,7 @@ import pypy -pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypydir = pypy.pypydir pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,7 +1,7 @@ import os import sys import py -from pypy.conftest import pypydir +from pypy import pypydir from rpython.tool.udir import udir if os.name != 'posix': diff --git a/pypy/tool/genstatistic.py b/pypy/tool/genstatistic.py --- a/pypy/tool/genstatistic.py +++ b/pypy/tool/genstatistic.py @@ -1,22 +1,22 @@ import py -from py._cmdline import pycountloc as countloc +from py._cmdline import pycountloc as countloc from py.xml import raw -from pypy import conftest +from pypy import pypydir -pypydir = py.path.local(conftest.pypydir) +pypydir = py.path.local(pypydir) def isdocfile(p): return (p.ext in ('.txt', '.rst') or p.basename in ('README', 'NOTES', 'LICENSE')) def istestfile(p): - if not p.check(file=1, ext='.py'): - return False - pb = p.purebasename - if pb.startswith('test_') or pb.endswith('_test'): - return True - if 'test' in [x.basename for x in p.parts()[-4:]]: + if not p.check(file=1, ext='.py'): + return False + pb = p.purebasename + if pb.startswith('test_') or pb.endswith('_test'): + return True + if 'test' in [x.basename for x in p.parts()[-4:]]: return True notistestfile = lambda x: not istestfile(x) @@ -24,42 +24,43 @@ class relchecker: def __init__(self, rel): self.rel = rel - def __call__(self, p): - return p.relto(conftest.pypydir).startswith(self.rel) + + def __call__(self, p): + return p.relto(pypydir).startswith(self.rel) def isfile(p): return p.check(file=1) and p.ext in ('.py', '.txt', '') def recpypy(p): - if p.basename[0] == '.': - return False - if p.basename in ('Pyrex', - '_cache', - 'unicodedata', + if p.basename[0] == '.': + return False + if p.basename in ('Pyrex', + '_cache', + 'unicodedata', 'pypy-translation-snapshot'): - return False - return True + return False + return True def getpypycounter(): - filecounter = countloc.FileCounter() - root = py.path.local(conftest.pypydir) + filecounter = countloc.FileCounter() + root = py.path.local(pypydir) filecounter.addrecursive(root, isfile, rec=recpypy) - return filecounter + return filecounter -class CounterModel: - def __init__(self, pypycounter): - self.counter = pypycounter - self.totallines = pypycounter.numlines +class CounterModel: + def __init__(self, pypycounter): + self.counter = pypycounter + self.totallines = pypycounter.numlines self.totalfiles = pypycounter.numfiles - self.testlines = pypycounter.getnumlines(istestfile) - self.testfiles = pypycounter.getnumfiles(istestfile) - self.notestlines = pypycounter.getnumlines(notistestfile) - self.notestfiles = pypycounter.getnumfiles(notistestfile) + self.testlines = pypycounter.getnumlines(istestfile) + self.testfiles = pypycounter.getnumfiles(istestfile) + self.notestlines = pypycounter.getnumlines(notistestfile) + self.notestfiles = pypycounter.getnumfiles(notistestfile) self.doclines = pypycounter.getnumlines(isdocfile) - self.docfiles = pypycounter.getnumfiles(isdocfile) + self.docfiles = pypycounter.getnumfiles(isdocfile) # -# rendering +# rendering # def row(*args): return html.tr([html.td(arg) for arg in args]) @@ -69,22 +70,22 @@ def viewlocsummary(model): t = html.table( - row("total number of lines", model.totallines, raw(" ")), - row("number of testlines", model.testlines, - percent(model.testlines, model.totallines)), - row("number of non-testlines", model.notestlines, - percent(model.notestlines, model.totallines)), + row("total number of lines", model.totallines, raw(" ")), + row("number of testlines", model.testlines, + percent(model.testlines, model.totallines)), + row("number of non-testlines", model.notestlines, + percent(model.notestlines, model.totallines)), - row("total number of files", model.totalfiles, raw(" ")), - row("number of testfiles", model.testfiles, - percent(model.testfiles, model.totalfiles)), - row("number of non-testfiles", model.notestfiles, - percent(model.notestfiles, model.totalfiles)), + row("total number of files", model.totalfiles, raw(" ")), + row("number of testfiles", model.testfiles, + percent(model.testfiles, model.totalfiles)), + row("number of non-testfiles", model.notestfiles, + percent(model.notestfiles, model.totalfiles)), ) - if model.docfiles: - t.append(row("number of docfiles", model.docfiles, + if model.docfiles: + t.append(row("number of docfiles", model.docfiles, percent(model.docfiles, model.totalfiles))) - t.append(row("number of doclines", model.doclines, + t.append(row("number of doclines", model.doclines, percent(model.doclines, model.totallines))) return t @@ -92,46 +93,46 @@ t = html.table() d = model.counter.file2numlines paths = d.items() - paths.sort(lambda x,y : -cmp(x[1], y[1])) # sort by numlines - for p, numlines in paths: - if numlines < 3: + paths.sort(lambda x, y: -cmp(x[1], y[1])) # sort by numlines + for p, numlines in paths: + if numlines < 3: continue t.append(row(p.relto(pypydir.dirpath()), numlines)) return t -def viewsubdirs(model): +def viewsubdirs(model): t = html.table() - for p in pypydir.listdir(): - if p.basename in '_cache .svn'.split(): + for p in pypydir.listdir(): + if p.basename in '_cache .svn'.split(): continue - if p.check(dir=1): + if p.check(dir=1): counter = countloc.FileCounter() counter.addrecursive(p, isfile, recpypy) - model = CounterModel(counter) + model = CounterModel(counter) t.append(row(html.h2(p.relto(pypydir.dirpath())))) t.append(viewlocsummary(model)) t.append(viewloclist(model)) return t -if __name__ == '__main__': +if __name__ == '__main__': if len(py.std.sys.argv) >= 2: target = py.path.local(py.std.sys.argv[1]) else: target = py.path.local('index.html') print "writing source statistics to", target - pypycounter = getpypycounter() - model = CounterModel(pypycounter) - rev = py.path.svnwc(conftest.pypydir).info().rev + pypycounter = getpypycounter() + model = CounterModel(pypycounter) + rev = py.path.svnwc(pypydir).info().rev html = py.xml.html doc = html.html( html.head( html.title("PyPy Statistics %d" % rev), - ), + ), html.body( html.h2("rev %d PyPy Summary of Files and Lines" % rev), - viewlocsummary(model), - html.h2("Details on first-level subdirectories"), - viewsubdirs(model), + viewlocsummary(model), + html.h2("Details on first-level subdirectories"), + viewsubdirs(model), html.h3("PyPy Full List Files and Lines"), viewloclist(model), html.p("files with less than 3 lines ignored") @@ -139,4 +140,3 @@ ) content = doc.unicode(indent=2).encode('utf8') target.write(content) - diff --git a/pypy/tool/getdocstrings.py b/pypy/tool/getdocstrings.py --- a/pypy/tool/getdocstrings.py +++ b/pypy/tool/getdocstrings.py @@ -1,7 +1,7 @@ import re from os import listdir from sys import stdin, stdout, stderr -from pypy.conftest import pypydir +from pypy import pypydir where = pypydir + '/objspace/std/' quote = '(' + "'" + '|' + '"' + ')' @@ -29,7 +29,7 @@ def compile_typedef(typ): return re.compile(r"(?P\s+)" - + r"(?P" + typ + + r"(?P" + typ + "_typedef = StdTypeDef+\s*\(\s*" + quote + typ + quote + ",).*" + r"(?P^\s+)" @@ -38,7 +38,7 @@ def get_pypydoc(sourcefile): doc = compile_doc() - + try: # if this works we already have a docstring pypydoc = doc.search(sourcefile).group('docstring') @@ -86,14 +86,10 @@ if __name__ == '__main__': filenames = mk_std_filelist() - + for f in filenames: inf = file(where + f).read() outs = add_docstring(f[:-7], inf) if outs is not None: outf = file(where + f, 'w') outf.write(outs) - - - - diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -1,6 +1,6 @@ import py -from pypy.conftest import pypydir +from pypy import pypydir from pypy.tool.release import package from pypy.module.sys.version import CPYTHON_VERSION from rpython.tool.udir import udir diff --git a/pypy/tool/test/test_tab.py b/pypy/tool/test/test_tab.py --- a/pypy/tool/test/test_tab.py +++ b/pypy/tool/test/test_tab.py @@ -3,7 +3,7 @@ """ import os -from pypy.conftest import pypydir +from pypy import pypydir ROOT = os.path.abspath(os.path.join(pypydir, '..')) RPYTHONDIR = os.path.join(ROOT, "rpython") From pypy.commits at gmail.com Sat Jun 4 11:58:00 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 04 Jun 2016 08:58:00 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Define missing _compute methods in assemble, fix bug of last commit in codegen Message-ID: <5752fa88.e4b3c20a.35934.fffff50c@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84918:ffef1437c3dc Date: 2016-06-04 17:57 +0200 http://bitbucket.org/pypy/pypy/changeset/ffef1437c3dc/ Log: Define missing _compute methods in assemble, fix bug of last commit in codegen diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -670,9 +670,15 @@ def _compute_BUILD_TUPLE(arg): return 1 - arg +def _compute_BUILD_TUPLE_UNPACK(arg): + return 1 - arg + def _compute_BUILD_LIST(arg): return 1 - arg +def _compute_BUILD_LIST_UNPACK(arg): + return 1 - arg + def _compute_BUILD_SET(arg): return 1 - arg @@ -685,6 +691,9 @@ def _compute_BUILD_MAP_UNPACK(arg): return 1 - arg +def _compute_BUILD_MAP_UNPACK_WITH_CALL(arg): + return 1 - (arg & 0xFF) + def _compute_MAKE_CLOSURE(arg): return -2 - _num_args(arg) - ((arg >> 16) & 0xFFFF) diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1113,7 +1113,7 @@ nkw = 0 nseen = 0 # the number of positional arguments on the stack for elt in args: - if isinstance(elt.func, ast.Starred): + if isinstance(elt, ast.Starred): # A star-arg. If we've seen positional arguments, # pack the positional arguments into a tuple. if nseen != 0: From pypy.commits at gmail.com Sat Jun 4 14:23:09 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:09 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head a0123b0e67f5 on branch s390x-enhance-speed Message-ID: <57531c8d.06321c0a.d5f2.ffffde56@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84922:77e48408589c Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/77e48408589c/ Log: Merge closed head a0123b0e67f5 on branch s390x-enhance-speed From pypy.commits at gmail.com Sat Jun 4 14:23:03 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:03 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 23e336d3e045 on branch bigint-with-int-ops Message-ID: <57531c87.08371c0a.7877d.ffffe385@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84919:c72419ba9c5a Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/c72419ba9c5a/ Log: Merge closed head 23e336d3e045 on branch bigint-with-int-ops From pypy.commits at gmail.com Sat Jun 4 14:23:06 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:06 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 1cd36e0809b5 on branch memop-simplify3 Message-ID: <57531c8a.89cbc20a.154d3.1de7@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84920:b58de1443ff9 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/b58de1443ff9/ Log: Merge closed head 1cd36e0809b5 on branch memop-simplify3 From pypy.commits at gmail.com Sat Jun 4 14:23:08 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:08 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 9a366b997dcc on branch s390x-backend Message-ID: <57531c8c.442cc20a.fc766.21b6@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84921:bab9605b99c7 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/bab9605b99c7/ Log: Merge closed head 9a366b997dcc on branch s390x-backend From pypy.commits at gmail.com Sat Jun 4 14:23:15 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:15 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 8520044ca6a8 on branch py3.3-hashfix Message-ID: <57531c93.665ec20a.79856.1e94@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84925:8167d3afbedd Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/8167d3afbedd/ Log: Merge closed head 8520044ca6a8 on branch py3.3-hashfix From pypy.commits at gmail.com Sat Jun 4 14:23:16 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:16 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 71ed4229465d on branch cpyext-macros-cast2 Message-ID: <57531c94.2a22c20a.694ef.150f@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84926:f96161023e92 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/f96161023e92/ Log: Merge closed head 71ed4229465d on branch cpyext-macros-cast2 From pypy.commits at gmail.com Sat Jun 4 14:23:11 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:11 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 0067281a14da on branch py3.3 Message-ID: <57531c8f.e7c9c20a.47bae.1b33@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84923:67d149a18183 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/67d149a18183/ Log: Merge closed head 0067281a14da on branch py3.3 From pypy.commits at gmail.com Sat Jun 4 14:23:13 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:13 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 2c67ae9165f5 on branch s390x-z196 Message-ID: <57531c91.c8e51c0a.db60c.ffffe032@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84924:229313695a85 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/229313695a85/ Log: Merge closed head 2c67ae9165f5 on branch s390x-z196 From pypy.commits at gmail.com Sat Jun 4 14:23:18 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:18 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head a33dbfaaca10 on branch py3k-clock_get_info Message-ID: <57531c96.e505c20a.6fe85.2382@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84927:cd27de52ceb5 Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/cd27de52ceb5/ Log: Merge closed head a33dbfaaca10 on branch py3k-clock_get_info From pypy.commits at gmail.com Sat Jun 4 14:23:20 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Jun 2016 11:23:20 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: re-close this branch Message-ID: <57531c98.230ec20a.83572.1bd1@mx.google.com> Author: Armin Rigo Branch: closed-branches Changeset: r84928:a0bfe513a90e Date: 2016-06-04 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/a0bfe513a90e/ Log: re-close this branch From pypy.commits at gmail.com Sat Jun 4 14:39:04 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 04 Jun 2016 11:39:04 -0700 (PDT) Subject: [pypy-commit] pypy default: fix const char* in function declaration Message-ID: <57532048.2450c20a.b8f3a.2516@mx.google.com> Author: Matti Picus Branch: Changeset: r84929:17d0560c9dda Date: 2016-06-04 21:38 +0300 http://bitbucket.org/pypy/pypy/changeset/17d0560c9dda/ Log: fix const char* in function declaration diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -85,7 +85,7 @@ w_buffer = space.call_function(space.w_bytearray, w_obj) return make_ref(space, w_buffer) - at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True) + at cpython_api([CONST_STRING, Py_ssize_t], PyObject, result_is_ll=True) def PyByteArray_FromStringAndSize(space, char_p, length): """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -104,7 +104,7 @@ PyObject* s1 = PyByteArray_FromStringAndSize("test", 4); if (s1 == NULL) return NULL; - char* c = PyByteArray_AsString(s1); + const char* c = PyByteArray_AsString(s1); PyObject* s2 = PyByteArray_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; From pypy.commits at gmail.com Sat Jun 4 17:33:34 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 04 Jun 2016 14:33:34 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Create unfinished _visit_starunpack, fix wrong methods bugs, fix None not iterable bug (still has bugs) Message-ID: <5753492e.43921c0a.2b3ee.1c53@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84930:6fe1da3c798a Date: 2016-06-04 23:32 +0200 http://bitbucket.org/pypy/pypy/changeset/6fe1da3c798a/ Log: Create unfinished _visit_starunpack, fix wrong methods bugs, fix None not iterable bug (still has bugs) diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1046,6 +1046,30 @@ self.visit_sequence(elts) if ctx == ast.Load: self.emit_op_arg(op, elt_count) + + #TODO + def _visit_starunpack(self, node, elts, ctx, single_op, innter_op, outer_op): + elt_count = len(elts) if elts else 0 + if ctx == ast.Store: + seen_star = False + for i in range(elt_count): + elt = elts[i] + is_starred = isinstance(elt, ast.Starred) + if is_starred and not seen_star: + if i >= 1 << 8 or elt_count - i - 1 >= (C_INT_MAX >> 8): + self.error("too many expressions in star-unpacking " + "assignment", node) + self.emit_op_arg(ops.UNPACK_EX, + i + ((elt_count - i - 1) << 8)) + seen_star = True + elts[i] = elt.value + elif is_starred: + self.error("two starred expressions in assignment", node) + if not seen_star: + self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count) + self.visit_sequence(elts) + if ctx == ast.Load: + self.emit_op_arg(op, elt_count) def visit_Starred(self, star): if star.ctx != ast.Store: @@ -1120,17 +1144,17 @@ ops.BUILD_TUPLE(nseen) nseen = 0 nsubargs += 1 - self.visit(elt.value) # probably wrong, elt->v.Starred.value + elt.walkabout(self) #self.visit(elt.value) # probably wrong, elt->v.Starred.value nsubargs += 1 elif nsubargs != 0: # We've seen star-args already, so we # count towards items-to-pack-into-tuple. - self.visit(elt) + elt.walkabout(self) nseen += 1 else: # Positional arguments before star-arguments # are left on the stack. - self.visit(elt) + elt.walkabout(self) n += 1 if nseen != 0: # Pack up any trailing positional arguments. @@ -1145,34 +1169,35 @@ # Repeat procedure for keyword args nseen = 0 # the number of keyword arguments on the stack following - for kw in keywords: - if kw.arg is None: - # A keyword argument unpacking. - if nseen != 0: - ops.BUILD_MAP(nseen) - nseen = 0 + if keywords is not None: + for kw in keywords: + if kw.arg is None: + # A keyword argument unpacking. + if nseen != 0: + ops.BUILD_MAP(nseen) + nseen = 0 + nsubkwargs += 1 + self.visit_sequence(kw.value) # probably wrong, elt->v.Starred.value nsubkwargs += 1 - self.visit(kw.value) # probably wrong, elt->v.Starred.value + elif nsubkwargs != 0: + # A keyword argument and we already have a dict. + ops.LOAD_CONST(kw.arg, consts) + self.visit_sequence(kw.value) + nseen += 1 + else: + # keyword argument + self.visit_sequence(kw) + nkw += 1 + if nseen != 0: + # Pack up any trailing keyword arguments. + ops.BUILD_MAP(nseen) nsubkwargs += 1 - elif nsubkwargs != 0: - # A keyword argument and we already have a dict. - ops.LOAD_CONST(kw.arg, consts) - self.visit(kw.value) - nseen += 1 - else: - # keyword argument - self.visit(kw) - nkw += 1 - if nseen != 0: - # Pack up any trailing keyword arguments. - ops.BUILD_MAP(nseen) - nsubkwargs += 1 - if nsubargs != 0: - call_type |= 2 - if nsubkwargs > 1: - # Pack it all up - function_pos = n + (code & 1) + nkw + 1 - ops.BUILD_MAP_UNPACK_WITH_CALL(nsubkwargs | (function_pos << 8)) + if nsubargs != 0: + call_type |= 2 + if nsubkwargs > 1: + # Pack it all up + function_pos = n + (code & 1) + nkw + 1 + ops.BUILD_MAP_UNPACK_WITH_CALL(nsubkwargs | (function_pos << 8)) assert n < 1<<8 assert nkw < 1<<24 From pypy.commits at gmail.com Sun Jun 5 04:09:41 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 01:09:41 -0700 (PDT) Subject: [pypy-commit] pypy default: 32-bit fix Message-ID: <5753de45.071d1c0a.51556.ffffa9b0@mx.google.com> Author: Armin Rigo Branch: Changeset: r84931:50777dc45a59 Date: 2016-06-05 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/50777dc45a59/ Log: 32-bit fix diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -30,7 +30,8 @@ #endif if(s->ob_type->tp_basicsize != expected_size) { - printf("tp_basicsize==%ld\\n", s->ob_type->tp_basicsize); + printf("tp_basicsize==%ld\\n", + (long)s->ob_type->tp_basicsize); result = 0; } Py_DECREF(s); From pypy.commits at gmail.com Sun Jun 5 06:22:52 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 03:22:52 -0700 (PDT) Subject: [pypy-commit] pypy default: Bogus interaction with option.keyword: running Message-ID: <5753fd7c.4e0b1c0a.8ba14.ffffd9c5@mx.google.com> Author: Armin Rigo Branch: Changeset: r84932:ac692f14d8b4 Date: 2016-06-05 12:08 +0200 http://bitbucket.org/pypy/pypy/changeset/ac692f14d8b4/ Log: Bogus interaction with option.keyword: running py.test test_newgc.py -k test_foo: would run only foo and skip all other tests diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -69,7 +69,7 @@ if not fullname.startswith('define'): continue keyword = option.keyword - if keyword.startswith('test_'): + if keyword.startswith('test_') and not keyword.endswith(':'): keyword = keyword[len('test_'):] if keyword not in fullname: continue @@ -93,6 +93,7 @@ funcs1.append(func) assert name not in name_to_func name_to_func[name] = len(name_to_func) + assert name_to_func def allfuncs(name, arg): num = name_to_func[name] From pypy.commits at gmail.com Sun Jun 5 06:23:49 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 03:23:49 -0700 (PDT) Subject: [pypy-commit] pypy default: In the C entrypoint, don't make RPython strings for argv. Instead, call Message-ID: <5753fdb5.4fa51c0a.d046d.ffffde1a@mx.google.com> Author: Armin Rigo Branch: Changeset: r84933:44f4010275d1 Date: 2016-06-05 12:23 +0200 http://bitbucket.org/pypy/pypy/changeset/44f4010275d1/ Log: In the C entrypoint, don't make RPython strings for argv. Instead, call an RPython function with the "char **argv" argument. Simplifies a deprecated bit of genc. (Copied from stmgc-c8 with various test fixes. It was also attempted on the refactor-translator branch, which was abandoned, but trying here to keep close to stmgc-c8 instead.) diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py --- a/rpython/memory/gctransform/test/test_framework.py +++ b/rpython/memory/gctransform/test/test_framework.py @@ -40,6 +40,7 @@ t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) + cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph @@ -115,6 +116,7 @@ t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) + cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() def test_no_collect_detection(): @@ -139,6 +141,7 @@ t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) + cbuild.make_entrypoint_wrapper = False with py.test.raises(Exception) as f: cbuild.build_database() expected = "'no_collect' function can trigger collection: char*, bool, bool # Can't inline this because of the raw address manipulation. diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -7,48 +7,15 @@ from rpython.translator.c.support import cdecl -def find_list_of_str(rtyper): - r_strlist = rtyper.getrepr(s_list_of_strings) - rtyper.call_all_setups() - return r_strlist.lowleveltype.TO - - def predeclare_common_types(db, rtyper): # Common types yield ('RPyString', STR) - LIST_OF_STR = find_list_of_str(rtyper) - yield ('RPyListOfString', LIST_OF_STR) def predeclare_utility_functions(db, rtyper): # Common utility functions def RPyString_New(length=lltype.Signed): return mallocstr(length) - # !!! - # be extremely careful passing a gc tracked object - # from such an helper result to another one - # as argument, this could result in leaks - # Such result should be only from C code - # returned directly as results - - LIST_OF_STR = find_list_of_str(rtyper) - p = lltype.Ptr(LIST_OF_STR) - - def _RPyListOfString_New(length=lltype.Signed): - return LIST_OF_STR.ll_newlist(length) - - def _RPyListOfString_SetItem(l=p, - index=lltype.Signed, - newstring=lltype.Ptr(STR)): - rlist.ll_setitem_nonneg(rlist.dum_nocheck, l, index, newstring) - - def _RPyListOfString_GetItem(l=p, - index=lltype.Signed): - return rlist.ll_getitem_fast(l, index) - - def _RPyListOfString_Length(l=p): - return rlist.ll_length(l) - for fname, f in locals().items(): if isinstance(f, types.FunctionType): # XXX this is painful :( diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -3,7 +3,7 @@ import sys, os from rpython.rlib import exports from rpython.rtyper.lltypesystem.lltype import getfunctionptr -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype from rpython.tool import runsubprocess from rpython.tool.nullpath import NullPyPathLocal from rpython.tool.udir import udir @@ -252,6 +252,8 @@ split = True executable_name = None shared_library_name = None + _entrypoint_wrapper = None + make_entrypoint_wrapper = True # for tests def getprofbased(self): profbased = None @@ -277,8 +279,39 @@ def getentrypointptr(self): # XXX check that the entrypoint has the correct # signature: list-of-strings -> int - bk = self.translator.annotator.bookkeeper - return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) + if not self.make_entrypoint_wrapper: + bk = self.translator.annotator.bookkeeper + return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) + if self._entrypoint_wrapper is not None: + return self._entrypoint_wrapper + # + from rpython.annotator import model as annmodel + from rpython.rtyper.lltypesystem import rffi + from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator + from rpython.rtyper.llannotation import lltype_to_annotation + entrypoint = self.entrypoint + # + def entrypoint_wrapper(argc, argv): + """This is a wrapper that takes "Signed argc" and "char **argv" + like the C main function, and puts them inside an RPython list + of strings before invoking the real entrypoint() function. + """ + list = [""] * argc + i = 0 + while i < argc: + list[i] = rffi.charp2str(argv[i]) + i += 1 + return entrypoint(list) + # + mix = MixLevelHelperAnnotator(self.translator.rtyper) + args_s = [annmodel.SomeInteger(), + lltype_to_annotation(rffi.CCHARPP)] + s_result = annmodel.SomeInteger() + graph = mix.getgraph(entrypoint_wrapper, args_s, s_result) + mix.finish() + res = getfunctionptr(graph) + self._entrypoint_wrapper = res + return res def cmdexec(self, args='', env=None, err=False, expect_crash=False, exe=None): assert self._compiled diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -62,7 +62,6 @@ { char *errmsg; int i, exitcode; - RPyListOfString *list; #if defined(MS_WINDOWS) && defined(RPY_SANDBOXED) _setmode(0, _O_BINARY); @@ -94,15 +93,7 @@ RPython_StartupCode(); - list = _RPyListOfString_New(argc); - if (RPyExceptionOccurred()) goto memory_out; - for (i=0; i= 4 - l0, l1, l2 = lines[-4:-1] + assert len(lines) >= 5 + l0, lx, l1, l2 = lines[-5:-1] assert l0 == 'RPython traceback:' + # lx is a bit strange with reference counting, ignoring it assert re.match(r' File "\w+.c", line \d+, in entry_point', l1) assert re.match(r' File "\w+.c", line \d+, in g', l2) # @@ -632,8 +633,9 @@ lines2 = err2.strip().splitlines() idx = lines2.index('Fatal RPython error: KeyError') # assert found lines2 = lines2[:idx+1] - l0, l1, l2 = lines2[-4:-1] + l0, lx, l1, l2 = lines2[-5:-1] assert l0 == 'RPython traceback:' + # lx is a bit strange with reference counting, ignoring it assert re.match(r' File "\w+.c", line \d+, in entry_point', l1) assert re.match(r' File "\w+.c", line \d+, in g', l2) assert lines2[-2] != lines[-2] # different line number @@ -660,9 +662,10 @@ lines = err.strip().splitlines() idx = lines.index('Fatal RPython error: KeyError') # assert found lines = lines[:idx+1] - assert len(lines) >= 5 - l0, l1, l2, l3 = lines[-5:-1] + assert len(lines) >= 6 + l0, lx, l1, l2, l3 = lines[-6:-1] assert l0 == 'RPython traceback:' + # lx is a bit strange with reference counting, ignoring it assert re.match(r' File "\w+.c", line \d+, in entry_point', l1) assert re.match(r' File "\w+.c", line \d+, in h', l2) assert re.match(r' File "\w+.c", line \d+, in g', l3) @@ -697,9 +700,10 @@ lines = err.strip().splitlines() idx = lines.index('Fatal RPython error: KeyError') # assert found lines = lines[:idx+1] - assert len(lines) >= 5 - l0, l1, l2, l3 = lines[-5:-1] + assert len(lines) >= 6 + l0, lx, l1, l2, l3 = lines[-6:-1] assert l0 == 'RPython traceback:' + # lx is a bit strange with reference counting, ignoring it assert re.match(r' File "\w+.c", line \d+, in entry_point', l1) assert re.match(r' File "\w+.c", line \d+, in h', l2) assert re.match(r' File "\w+.c", line \d+, in g', l3) @@ -733,9 +737,10 @@ lines = err.strip().splitlines() idx = lines.index('Fatal RPython error: ValueError') # assert found lines = lines[:idx+1] - assert len(lines) >= 5 - l0, l1, l2, l3 = lines[-5:-1] + assert len(lines) >= 6 + l0, lx, l1, l2, l3 = lines[-6:-1] assert l0 == 'RPython traceback:' + # lx is a bit strange with reference counting, ignoring it assert re.match(r' File "\w+.c", line \d+, in entry_point', l1) assert re.match(r' File "\w+.c", line \d+, in h', l2) assert re.match(r' File "\w+.c", line \d+, in raiseme', l3) From pypy.commits at gmail.com Sun Jun 5 08:20:34 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 05:20:34 -0700 (PDT) Subject: [pypy-commit] pypy default: Documentation for ll_nonmovable_raw_ptr_for_resizable_list() Message-ID: <57541912.d81b1c0a.2d550.fffffea3@mx.google.com> Author: Armin Rigo Branch: Changeset: r84934:6aeeb58ecd40 Date: 2016-06-05 14:21 +0200 http://bitbucket.org/pypy/pypy/changeset/6aeeb58ecd40/ Log: Documentation for ll_nonmovable_raw_ptr_for_resizable_list() diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1242,6 +1242,19 @@ v_list) def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): + """ + WARNING: dragons ahead. + Return the address of the internal char* buffer of 'll_list', which + must be a resizable list of chars. + + This makes sure that the list items are non-moving, if necessary by + first copying the GcArray inside 'll_list.items' outside the GC + nursery. The returned 'char *' pointer is guaranteed to be valid + until one of these occurs: + + * 'll_list' gets garbage-collected; or + * you do an operation on 'll_list' that changes its size. + """ from rpython.rtyper.lltypesystem import lltype, rffi array = ll_list.items if can_move(array): From pypy.commits at gmail.com Sun Jun 5 08:23:22 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 05:23:22 -0700 (PDT) Subject: [pypy-commit] pypy default: Use ll_arraycopy() instead of looping Message-ID: <575419ba.230ec20a.83572.3d15@mx.google.com> Author: Armin Rigo Branch: Changeset: r84935:da5b62f5d057 Date: 2016-06-05 14:24 +0200 http://bitbucket.org/pypy/pypy/changeset/da5b62f5d057/ Log: Use ll_arraycopy() instead of looping diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1261,10 +1261,7 @@ length = ll_list.length new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length, nonmovable=True) - i = 0 - while i < length: - new_array[i] = array[i] - i += 1 + ll_arraycopy(array, new_array, 0, 0, length) ll_list.items = new_array array = new_array ptr = lltype.direct_arrayitems(array) From pypy.commits at gmail.com Sun Jun 5 10:10:56 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 07:10:56 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: A branch to play with the idea of a reverse debugger Message-ID: <575432f0.2a22c20a.694ef.5673@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84936:086e9a6ea427 Date: 2016-06-05 14:31 +0200 http://bitbucket.org/pypy/pypy/changeset/086e9a6ea427/ Log: A branch to play with the idea of a reverse debugger From pypy.commits at gmail.com Sun Jun 5 10:10:58 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 07:10:58 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Copy from stmgc-c8 the logic to disable some fast paths that read or Message-ID: <575432f2.891ec20a.11f71.5778@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84937:bbd0913d7575 Date: 2016-06-05 16:11 +0200 http://bitbucket.org/pypy/pypy/changeset/bbd0913d7575/ Log: Copy from stmgc-c8 the logic to disable some fast paths that read or write inside GC objects (like strings) in bulk. This makes the first test in 'reversedb' pass. diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -276,6 +276,13 @@ suggests={"arm": [("translation.gcrootfinder", "shadowstack"), ("translation.jit_backend", "arm")]}), + BoolOption("split_gc_address_space", + "Ensure full separation of GC and non-GC pointers", default=False), + BoolOption("reversedb", + "Give an executable that writes a log file for reverse debugging", + default=False, cmdline='--reversedb', + requires=[('translation.split_gc_address_space', True), + ('translation.jit', False)]), ]) def get_combined_translation_config(other_optdescr=None, diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -17,11 +17,11 @@ self.finalizer_funcptrs = {} atomic_mh = mallocHelpers() - atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) + atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.GCREF, size) ll_malloc_fixedsize_atomic = atomic_mh._ll_malloc_fixedsize mh = mallocHelpers() - mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size) + mh.allocate = lambda size: llop.boehm_malloc(llmemory.GCREF, size) ll_malloc_fixedsize = mh._ll_malloc_fixedsize # XXX, do we need/want an atomic version of this function? @@ -39,13 +39,13 @@ if self.translator: self.malloc_fixedsize_ptr = self.inittime_helper( - ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) + ll_malloc_fixedsize, [lltype.Signed], llmemory.GCREF) self.malloc_fixedsize_atomic_ptr = self.inittime_helper( - ll_malloc_fixedsize_atomic, [lltype.Signed], llmemory.Address) + ll_malloc_fixedsize_atomic, [lltype.Signed], llmemory.GCREF) self.malloc_varsize_no_length_ptr = self.inittime_helper( - ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) + ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.GCREF, inline=False) self.malloc_varsize_ptr = self.inittime_helper( - ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address, inline=False) + ll_malloc_varsize, [lltype.Signed]*4, llmemory.GCREF, inline=False) if self.translator.config.translation.rweakref: self.weakref_create_ptr = self.inittime_helper( ll_weakref_create, [llmemory.Address], llmemory.WeakRefPtr, @@ -66,7 +66,7 @@ funcptr = self.malloc_fixedsize_ptr v_raw = hop.genop("direct_call", [funcptr, c_size], - resulttype=llmemory.Address) + resulttype=llmemory.GCREF) finalizer_ptr = self.finalizer_funcptr_for_type(TYPE) if finalizer_ptr: c_finalizer_ptr = Constant(finalizer_ptr, self.FINALIZER_PTR) @@ -80,12 +80,12 @@ v_raw = hop.genop("direct_call", [self.malloc_varsize_no_length_ptr, v_length, c_const_size, c_item_size], - resulttype=llmemory.Address) + resulttype=llmemory.GCREF) else: v_raw = hop.genop("direct_call", [self.malloc_varsize_ptr, v_length, c_const_size, c_item_size, c_offset_to_length], - resulttype=llmemory.Address) + resulttype=llmemory.GCREF) return v_raw def finalizer_funcptr_for_type(self, TYPE): diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -444,7 +444,7 @@ def ll_malloc_varsize(length, size, itemsize, lengthoffset): result = mh.ll_malloc_varsize_no_length(length, size, itemsize) - (result + lengthoffset).signed[0] = length + llop.raw_store(lltype.Void, result, lengthoffset, length) return result mh.ll_malloc_varsize = ll_malloc_varsize @@ -471,9 +471,9 @@ ll_raw_malloc_varsize = mh.ll_malloc_varsize ll_raw_malloc_varsize_no_length_zero = mh.ll_malloc_varsize_no_length_zero - stack_mh = mallocHelpers() - stack_mh.allocate = lambda size: llop.stack_malloc(llmemory.Address, size) - ll_stack_malloc_fixedsize = stack_mh._ll_malloc_fixedsize + ## stack_mh = mallocHelpers() + ## stack_mh.allocate = lambda size: llop.stack_malloc(llmemory.GCREF, size) + ## ll_stack_malloc_fixedsize = stack_mh._ll_malloc_fixedsize if self.translator: self.raw_malloc_fixedsize_ptr = self.inittime_helper( @@ -485,8 +485,8 @@ self.raw_malloc_varsize_no_length_zero_ptr = self.inittime_helper( ll_raw_malloc_varsize_no_length_zero, [lltype.Signed]*3, llmemory.Address, inline=False) - self.stack_malloc_fixedsize_ptr = self.inittime_helper( - ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) + ## self.stack_malloc_fixedsize_ptr = self.inittime_helper( + ## ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) def gct_malloc(self, hop, add_flags=None): TYPE = hop.spaceop.result.concretetype.TO @@ -509,11 +509,12 @@ return v_raw def gct_fv_stack_malloc(self, hop, flags, TYPE, c_size): - v_raw = hop.genop("direct_call", [self.stack_malloc_fixedsize_ptr, c_size], - resulttype=llmemory.Address) - if flags.get('zero'): - hop.genop("raw_memclear", [v_raw, c_size]) - return v_raw + raise Exception("not supported any more") + ## v_raw = hop.genop("direct_call", [self.stack_malloc_fixedsize_ptr, c_size], + ## resulttype=llmemory.Address) + ## if flags.get('zero'): + ## hop.genop("raw_memclear", [v_raw, c_size]) + ## return v_raw def gct_malloc_varsize(self, hop, add_flags=None): flags = hop.spaceop.args[1].value diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -19,6 +19,13 @@ """ pass +def must_split_gc_address_space(): + """Returns True if we have a "split GC address space", i.e. if + we are translating with an option that doesn't support taking raw + addresses inside GC objects and "hacking" at them. This is + notably the case with --reversedb.""" + return False + # for test purposes we allow objects to be pinned and use # the following list to keep track of the pinned objects _pinned_objects = [] @@ -147,6 +154,18 @@ """ return True +class SplitAddrSpaceEntry(ExtRegistryEntry): + _about_ = must_split_gc_address_space + + def compute_result_annotation(self): + config = self.bookkeeper.annotator.translator.config + result = config.translation.split_gc_address_space + return self.bookkeeper.immutablevalue(result) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputconst(lltype.Bool, hop.s_result.const) + class CanMoveEntry(ExtRegistryEntry): _about_ = can_move @@ -280,18 +299,25 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if _contains_gcptr(TP.OF): + + slowpath = False + if must_split_gc_address_space(): + slowpath = True + elif _contains_gcptr(TP.OF): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, source_start, dest_start, length): - # if the write barrier is not supported, copy by hand - i = 0 - while i < length: - copy_item(source, dest, i + source_start, i + dest_start) - i += 1 - return + slowpath = True + if slowpath: + # if the write barrier is not supported, or if we translate with + # the option 'split_gc_address_space', then copy by hand + i = 0 + while i < length: + copy_item(source, dest, i + source_start, i + dest_start) + i += 1 + return source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + @@ -325,6 +351,14 @@ field = getattr(p, TP._names[0]) setattr(newp, TP._names[0], field) + if must_split_gc_address_space(): + # do the copying element by element + i = 0 + while i < smallerlength: + newp.chars[i] = p.chars[i] + i += 1 + return newp + ARRAY = getattr(TP, TP._arrayfld) offset = (llmemory.offsetof(TP, TP._arrayfld) + llmemory.itemoffsetof(ARRAY, 0)) @@ -345,9 +379,18 @@ length = len(p) ARRAY = lltype.typeOf(p).TO - offset = llmemory.itemoffsetof(ARRAY, 0) - dest_addr = llmemory.cast_ptr_to_adr(p) + offset - llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length) + if must_split_gc_address_space(): + # do the clearing element by element + from rpython.rtyper.lltypesystem import rffi + ZERO = rffi.cast(ARRAY.OF, 0) + i = 0 + while i < length: + p[i] = ZERO + i += 1 + else: + offset = llmemory.itemoffsetof(ARRAY, 0) + dest_addr = llmemory.cast_ptr_to_adr(p) + offset + llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length) keepalive_until_here(p) diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -825,14 +825,19 @@ count = len(data) pinned = False - if rgc.can_move(data): + fallback = False + if rgc.must_split_gc_address_space(): + fallback = True + elif rgc.can_move(data): if rgc.pin(data): pinned = True else: - buf = lltype.malloc(TYPEP.TO, count, flavor='raw') - copy_string_to_raw(lldata, buf, 0, count) - return buf, pinned, True - # ^^^ raw malloc used to get a nonmovable copy + fallback = True + if fallback: + buf = lltype.malloc(TYPEP.TO, count, flavor='raw') + copy_string_to_raw(lldata, buf, 0, count) + return buf, pinned, True + # ^^^ raw malloc used to get a nonmovable copy # # following code is executed if: # - rgc.can_move(data) and rgc.pin(data) both returned true @@ -878,12 +883,17 @@ """ new_buf = mallocfn(count) pinned = 0 - if rgc.can_move(new_buf): + fallback = False + if rgc.must_split_gc_address_space(): + fallback = True + elif rgc.can_move(new_buf): if rgc.pin(new_buf): pinned = 1 else: - raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') - return raw_buf, new_buf, 2 + fallback = True + if fallback: + raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw') + return raw_buf, new_buf, 2 # # following code is executed if: # - rgc.can_move(data) and rgc.pin(data) both returned true diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -1,7 +1,7 @@ from weakref import WeakValueDictionary from rpython.annotator import model as annmodel -from rpython.rlib import jit, types +from rpython.rlib import jit, types, rgc from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated, _hash_string, keepalive_until_here, specialize, enforceargs) from rpython.rlib.signature import signature @@ -88,6 +88,17 @@ ll_assert(srcstart + length <= len(src.chars), "copystrc: src ovf") ll_assert(dststart >= 0, "copystrc: negative dststart") ll_assert(dststart + length <= len(dst.chars), "copystrc: dst ovf") + # + # If the 'split_gc_address_space' option is set, we must copy + # manually, character-by-character + if rgc.must_split_gc_address_space(): + i = 0 + while i < length: + dst.chars[dststart + i] = src.chars[srcstart + i] + i += 1 + return + # + # # from here, no GC operations can happen asrc = _get_raw_buf(SRC_TP, src, srcstart) adst = _get_raw_buf(DST_TP, dst, dststart) @@ -108,6 +119,16 @@ """ # xxx Warning: same note as above apply: don't do this at home assert length >= 0 + # + # If the 'split_gc_address_space' option is set, we must copy + # manually, character-by-character + if rgc.must_split_gc_address_space(): + i = 0 + while i < length: + ptrdst[i] = src.chars[srcstart + i] + i += 1 + return + # # from here, no GC operations can happen asrc = _get_raw_buf(SRC_TP, src, srcstart) adst = llmemory.cast_ptr_to_adr(ptrdst) @@ -124,6 +145,16 @@ def copy_raw_to_string(ptrsrc, dst, dststart, length): # xxx Warning: same note as above apply: don't do this at home assert length >= 0 + # + # If the 'split_gc_address_space' option is set, we must copy + # manually, character-by-character + if rgc.must_split_gc_address_space(): + i = 0 + while i < length: + dst.chars[dststart + i] = ptrsrc[i] + i += 1 + return + # # from here, no GC operations can happen adst = _get_raw_buf(SRC_TP, dst, dststart) asrc = llmemory.cast_ptr_to_adr(ptrsrc) @@ -1221,6 +1252,16 @@ SRC = typeOf(src).TO # STR or UNICODE DST = typeOf(dst).TO # GcArray assert DST.OF is SRC.chars.OF + # + # If the 'split_gc_address_space' option is set, we must copy + # manually, character-by-character + if rgc.must_split_gc_address_space(): + i = 0 + while i < length: + dst[i] = src.chars[i] + i += 1 + return lst + # # from here, no GC operations can happen asrc = llmemory.cast_ptr_to_adr(src) + ( llmemory.offsetof(SRC, 'chars') + diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -31,10 +31,12 @@ gcpolicyclass=None, exctransformer=None, thread_enabled=False, - sandbox=False): + sandbox=False, + split_gc_address_space=False): self.translator = translator self.standalone = standalone self.sandbox = sandbox + self.split_gc_address_space = split_gc_address_space if gcpolicyclass is None: gcpolicyclass = gc.RefcountingGcPolicy self.gcpolicy = gcpolicyclass(self, thread_enabled) diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -606,7 +606,23 @@ self.expr(op.args[0]), self.expr(op.args[1])) + def _check_split_gc_address_space(self, op): + if self.db.split_gc_address_space: + TYPE = self.lltypemap(op.result) + TSRC = self.lltypemap(op.args[0]) + gcdst = isinstance(TYPE, Ptr) and TYPE.TO._gckind == 'gc' + gcsrc = isinstance(TSRC, Ptr) and TSRC.TO._gckind == 'gc' + if gcsrc != gcdst: + raise Exception( + "cast between pointer types changes the address space,\n" + "but the 'split_gc_address_space' option is enabled:\n" + " func: %s\n" + " op: %s\n" + " from: %s\n" + " to: %s" % (self.graph, op, TSRC, TYPE)) + def OP_CAST_POINTER(self, op): + self._check_split_gc_address_space(op) TYPE = self.lltypemap(op.result) typename = self.db.gettype(TYPE) result = [] @@ -625,6 +641,7 @@ % (self.expr(op.result), self.expr(op.args[0]))) def OP_CAST_INT_TO_PTR(self, op): + self._check_split_gc_address_space(op) TYPE = self.lltypemap(op.result) typename = self.db.gettype(TYPE) return "%s = (%s)%s;" % (self.expr(op.result), cdecl(typename, ""), @@ -690,6 +707,7 @@ % locals()) def OP_CAST_PRIMITIVE(self, op): + self._check_split_gc_address_space(op) TYPE = self.lltypemap(op.result) val = self.expr(op.args[0]) result = self.expr(op.result) diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -131,7 +131,9 @@ gcpolicyclass=gcpolicyclass, exctransformer=exctransformer, thread_enabled=self.config.translation.thread, - sandbox=self.config.translation.sandbox) + sandbox=self.config.translation.sandbox, + split_gc_address_space= + self.config.translation.split_gc_address_space) self.db = db # give the gc a chance to register interest in the start-up functions it diff --git a/rpython/translator/reversedb/__init__.py b/rpython/translator/reversedb/__init__.py new file mode 100644 diff --git a/rpython/translator/reversedb/test/__init__.py b/rpython/translator/reversedb/test/__init__.py new file mode 100644 diff --git a/rpython/translator/reversedb/test/test_basic.py b/rpython/translator/reversedb/test/test_basic.py new file mode 100644 --- /dev/null +++ b/rpython/translator/reversedb/test/test_basic.py @@ -0,0 +1,29 @@ +import py +from rpython.translator.interactive import Translation + + +class TestBasic(object): + + def getcompiled(self, entry_point, argtypes, backendopt=True): + t = Translation(entry_point, None, gc="boehm") + t.config.translation.reversedb = True + t.config.translation.rweakref = False + if not backendopt: + t.disable(["backendopt_lltype"]) + t.annotate() + t.rtype() + if t.backendopt: + t.backendopt() + t.compile_c() + + def run(*argv): + stdout = t.driver.cbuilder.cmdexec(' '.join(argv)) + return stdout + return run + + def test_simple(self): + def main(argv): + print argv[1:] + return 0 + fn = self.getcompiled(main, [], backendopt=False) + assert fn('abc d') == '[abc, d]\n' From pypy.commits at gmail.com Sun Jun 5 11:14:55 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 08:14:55 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Start to write the logic to emit a log file Message-ID: <575441ef.0654c20a.c16a9.721e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84938:9b3d9afb873c Date: 2016-06-05 17:15 +0200 http://bitbucket.org/pypy/pypy/changeset/9b3d9afb873c/ Log: Start to write the logic to emit a log file diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -32,11 +32,13 @@ exctransformer=None, thread_enabled=False, sandbox=False, - split_gc_address_space=False): + split_gc_address_space=False, + reversedb=False): self.translator = translator self.standalone = standalone self.sandbox = sandbox self.split_gc_address_space = split_gc_address_space + self.reversedb = reversedb if gcpolicyclass is None: gcpolicyclass = gc.RefcountingGcPolicy self.gcpolicy = gcpolicyclass(self, thread_enabled) diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -2,6 +2,7 @@ from rpython.translator.c.support import cdecl from rpython.translator.c.support import llvalue_from_constant, gen_assignments from rpython.translator.c.support import c_string_constant, barebonearray +from rpython.translator.c.primitive import PRIMITIVE_FLOATS from rpython.flowspace.model import Variable, Constant from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned, SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType, @@ -424,6 +425,14 @@ def OP_JIT_CONDITIONAL_CALL(self, op): return 'abort(); /* jit_conditional_call */' + def _reverse_db_emit(self, T, value): + if T is Void: + return '/* rpy_reverse_db_emit_void(%s); */' % (value,) + elif T in PRIMITIVE_FLOATS: + return 'rpy_reverse_db_emit_float(%s);' % (value,) + else: + return 'rpy_reverse_db_emit((Signed)%s);' % (value,) + # low-level operations def generic_get(self, op, sourceexpr): T = self.lltypemap(op.result) @@ -431,6 +440,9 @@ result = '%s = %s;' % (newvalue, sourceexpr) if T is Void: result = '/* %s */' % result + if self.db.reversedb: + if self.lltypemap(op.args[0]).TO._gckind != 'gc': + result += '\t' + self._reverse_db_emit(T, newvalue) return result def generic_set(self, op, targetexpr): diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -115,6 +115,8 @@ def get_eci(self): pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] + if self.config.translation.reversedb: + include_dirs.append(pypy_include_dir.join('..', 'reversedb')) return ExternalCompilationInfo(include_dirs=include_dirs) def build_database(self): @@ -133,7 +135,8 @@ thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox, split_gc_address_space= - self.config.translation.split_gc_address_space) + self.config.translation.split_gc_address_space, + reversedb=self.config.translation.reversedb) self.db = db # give the gc a chance to register interest in the start-up functions it @@ -215,6 +218,8 @@ defines['COUNT_OP_MALLOCS'] = 1 if self.config.translation.sandbox: defines['RPY_SANDBOXED'] = 1 + if self.config.translation.reversedb: + defines['RPY_REVERSE_DB'] = 1 if CBuilder.have___thread is None: CBuilder.have___thread = self.translator.platform.check___thread() if not self.standalone: @@ -822,7 +827,7 @@ defines['PYPY_LONG_BIT'] = LONG_BIT defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT -def add_extra_files(eci): +def add_extra_files(database, eci): srcdir = py.path.local(__file__).join('..', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE @@ -841,6 +846,9 @@ ] if _CYGWIN: files.append(srcdir / 'cygwin_wait.c') + if database.reversedb: + from rpython.translator.reversedb import rdb_genc + files += rdb_genc.extra_files() return eci.merge(ExternalCompilationInfo(separate_module_files=files)) @@ -889,6 +897,6 @@ print >>fi, "#define PYPY_INSTRUMENT_NCOUNTER %d" % n fi.close() - eci = add_extra_files(eci) + eci = add_extra_files(database, eci) eci = eci.convert_sources_to_files() return eci, filename, sg.getextrafiles(), headers_to_precompile diff --git a/rpython/translator/c/primitive.py b/rpython/translator/c/primitive.py --- a/rpython/translator/c/primitive.py +++ b/rpython/translator/c/primitive.py @@ -223,6 +223,7 @@ Address: name_address, GCREF: name_gcref, } +PRIMITIVE_FLOATS = set([Float, SingleFloat, LongFloat]) PrimitiveType = { SignedLongLong: 'long long @', diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -82,6 +82,10 @@ pypy_asm_stack_bottom(); instrument_setup(); +#ifdef RPY_REVERSE_DB + rpy_reverse_db_setup(argc, argv); +#endif + #ifndef MS_WINDOWS /* this message does no longer apply to win64 :-) */ if (sizeof(void*) != SIZEOF_LONG) { diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h --- a/rpython/translator/c/src/g_include.h +++ b/rpython/translator/c/src/g_include.h @@ -50,3 +50,7 @@ #ifdef __CYGWIN__ #include "src/cygwin_wait.h" #endif + +#ifdef RPY_REVERSE_DB +#include "rdb-src/rdb_include.h" +#endif diff --git a/rpython/translator/reversedb/rdb-src/rdb.c b/rpython/translator/reversedb/rdb-src/rdb.c new file mode 100644 --- /dev/null +++ b/rpython/translator/reversedb/rdb-src/rdb.c @@ -0,0 +1,60 @@ +#include "common_header.h" +#include +#include +#include +#include + +#include "rdb-src/rdb_include.h" + +#define RDB_SIGNATURE 0x0A424452 /* "RDB\n" */ +#define RDB_VERSION 0x00FF0001 + + +Signed *rpy_rev_buf_p, *rpy_rev_buf_end; +static Signed rpy_rev_buffer[2048]; +static int rpy_rev_fileno = -1; + + +RPY_EXTERN +void rpy_reverse_db_setup(int argc, char *argv[]) +{ + /* init-time setup */ + + char *filename = getenv("PYPYRDB"); + if (filename && *filename) + putenv("PYPYRDB="); + else + filename = "/dev/null"; + + rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC | + O_CREAT | O_NOCTTY | O_TRUNC, 0600); + if (rpy_rev_fileno < 0) { + fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n", + filename); + abort(); + } + + rpy_rev_buf_p = rpy_rev_buffer; + rpy_rev_buf_end = rpy_rev_buffer + + sizeof(rpy_rev_buffer) / sizeof(rpy_rev_buffer[0]); + atexit(rpy_reverse_db_flush); + + rpy_reverse_db_emit(RDB_SIGNATURE); + rpy_reverse_db_emit(RDB_VERSION); + rpy_reverse_db_emit(0); + rpy_reverse_db_emit(0); + rpy_reverse_db_emit(argc); +} + +RPY_EXTERN +void rpy_reverse_db_flush(void) +{ + /* write the current buffer content to the OS */ + + ssize_t size = (rpy_rev_buf_p - rpy_rev_buffer) * sizeof(rpy_rev_buffer[0]); + rpy_rev_buf_p = rpy_rev_buffer; + if (size > 0 && write(rpy_rev_fileno, rpy_rev_buffer, size) != size) { + fprintf(stderr, "Fatal error: writing to PYPYRDB file: %m\n"); + abort(); + } +} diff --git a/rpython/translator/reversedb/rdb-src/rdb_include.h b/rpython/translator/reversedb/rdb-src/rdb_include.h new file mode 100644 --- /dev/null +++ b/rpython/translator/reversedb/rdb-src/rdb_include.h @@ -0,0 +1,23 @@ + + +RPY_EXTERN void rpy_reverse_db_setup(int argc, char *argv[]); +RPY_EXTERN void rpy_reverse_db_flush(void); + + +RPY_EXTERN Signed *rpy_rev_buf_p, *rpy_rev_buf_end; + + +static inline void rpy_reverse_db_emit(Signed value) { + *(rpy_rev_buf_p++) = value; + if (rpy_rev_buf_p == rpy_rev_buf_end) + rpy_reverse_db_flush(); +} +static inline void rpy_reverse_db_emit_float(double value) { + /* xxx for 'long double' this can loose some precision */ + Signed sval[8 / SIZEOF_LONG]; + assert(sizeof(double) == 8); + memcpy(sval, &value, sizeof(value)); + rpy_reverse_db_emit(sval[0]); + if (SIZEOF_LONG <= 4) + rpy_reverse_db_emit(sval[1]); +} diff --git a/rpython/translator/reversedb/rdb_genc.py b/rpython/translator/reversedb/rdb_genc.py new file mode 100644 --- /dev/null +++ b/rpython/translator/reversedb/rdb_genc.py @@ -0,0 +1,7 @@ +import py + +def extra_files(): + srcdir = py.path.local(__file__).join('..', 'rdb-src') + return [ + srcdir / 'rdb.c', + ] From pypy.commits at gmail.com Sun Jun 5 11:46:19 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 08:46:19 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <5754494b.cda91c0a.f49a4.4274@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84939:b7e8d7232049 Date: 2016-06-05 17:47 +0200 http://bitbucket.org/pypy/pypy/changeset/b7e8d7232049/ Log: in-progress diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -2,7 +2,7 @@ from rpython.translator.c.support import cdecl from rpython.translator.c.support import llvalue_from_constant, gen_assignments from rpython.translator.c.support import c_string_constant, barebonearray -from rpython.translator.c.primitive import PRIMITIVE_FLOATS +from rpython.translator.c.primitive import PRIMITIVE_FLOATS, PRIMITIVE_TWO_LONGS from rpython.flowspace.model import Variable, Constant from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned, SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType, @@ -430,6 +430,8 @@ return '/* rpy_reverse_db_emit_void(%s); */' % (value,) elif T in PRIMITIVE_FLOATS: return 'rpy_reverse_db_emit_float(%s);' % (value,) + elif T in PRIMITIVE_TWO_LONGS: + return 'rpy_reverse_db_emit_two_longs(%s);' % (value,) else: return 'rpy_reverse_db_emit((Signed)%s);' % (value,) @@ -441,7 +443,8 @@ if T is Void: result = '/* %s */' % result if self.db.reversedb: - if self.lltypemap(op.args[0]).TO._gckind != 'gc': + S = self.lltypemap(op.args[0]).TO + if S._gckind != 'gc' and not S._hints.get('is_excdata'): result += '\t' + self._reverse_db_emit(T, newvalue) return result diff --git a/rpython/translator/c/primitive.py b/rpython/translator/c/primitive.py --- a/rpython/translator/c/primitive.py +++ b/rpython/translator/c/primitive.py @@ -224,6 +224,10 @@ GCREF: name_gcref, } PRIMITIVE_FLOATS = set([Float, SingleFloat, LongFloat]) +if SignedLongLong is Signed: + PRIMITIVE_TWO_LONGS = set([]) +else: + PRIMITIVE_TWO_LONGS = set([SignedLongLong, UnsignedLongLong]) PrimitiveType = { SignedLongLong: 'long long @', diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -452,7 +452,8 @@ def setup_excdata(self): EXCDATA = lltype.Struct('ExcData', ('exc_type', self.lltype_of_exception_type), - ('exc_value', self.lltype_of_exception_value)) + ('exc_value', self.lltype_of_exception_value), + hints={'is_excdata': True}) self.EXCDATA = EXCDATA exc_data = lltype.malloc(EXCDATA, immortal=True) diff --git a/rpython/translator/reversedb/rdb-src/rdb.c b/rpython/translator/reversedb/rdb-src/rdb.c --- a/rpython/translator/reversedb/rdb-src/rdb.c +++ b/rpython/translator/reversedb/rdb-src/rdb.c @@ -21,29 +21,29 @@ /* init-time setup */ char *filename = getenv("PYPYRDB"); - if (filename && *filename) - putenv("PYPYRDB="); - else - filename = "/dev/null"; - - rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC | - O_CREAT | O_NOCTTY | O_TRUNC, 0600); - if (rpy_rev_fileno < 0) { - fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n", - filename); - abort(); - } rpy_rev_buf_p = rpy_rev_buffer; rpy_rev_buf_end = rpy_rev_buffer + sizeof(rpy_rev_buffer) / sizeof(rpy_rev_buffer[0]); - atexit(rpy_reverse_db_flush); + + if (filename && *filename) { + putenv("PYPYRDB="); + rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC | + O_CREAT | O_NOCTTY | O_TRUNC, 0600); + if (rpy_rev_fileno < 0) { + fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n", + filename); + abort(); + } + atexit(rpy_reverse_db_flush); + } rpy_reverse_db_emit(RDB_SIGNATURE); rpy_reverse_db_emit(RDB_VERSION); rpy_reverse_db_emit(0); rpy_reverse_db_emit(0); rpy_reverse_db_emit(argc); + rpy_reverse_db_emit((Signed)argv); } RPY_EXTERN @@ -53,8 +53,10 @@ ssize_t size = (rpy_rev_buf_p - rpy_rev_buffer) * sizeof(rpy_rev_buffer[0]); rpy_rev_buf_p = rpy_rev_buffer; - if (size > 0 && write(rpy_rev_fileno, rpy_rev_buffer, size) != size) { - fprintf(stderr, "Fatal error: writing to PYPYRDB file: %m\n"); - abort(); + if (size > 0 && rpy_rev_fileno >= 0) { + if (write(rpy_rev_fileno, rpy_rev_buffer, size) != size) { + fprintf(stderr, "Fatal error: writing to PYPYRDB file: %m\n"); + abort(); + } } } diff --git a/rpython/translator/reversedb/rdb-src/rdb_include.h b/rpython/translator/reversedb/rdb-src/rdb_include.h --- a/rpython/translator/reversedb/rdb-src/rdb_include.h +++ b/rpython/translator/reversedb/rdb-src/rdb_include.h @@ -1,4 +1,4 @@ - +#include RPY_EXTERN void rpy_reverse_db_setup(int argc, char *argv[]); RPY_EXTERN void rpy_reverse_db_flush(void); @@ -12,12 +12,21 @@ if (rpy_rev_buf_p == rpy_rev_buf_end) rpy_reverse_db_flush(); } + static inline void rpy_reverse_db_emit_float(double value) { /* xxx for 'long double' this can loose some precision */ - Signed sval[8 / SIZEOF_LONG]; - assert(sizeof(double) == 8); + Signed sval[sizeof(double) / SIZEOF_LONG]; memcpy(sval, &value, sizeof(value)); rpy_reverse_db_emit(sval[0]); - if (SIZEOF_LONG <= 4) + if (SIZEOF_LONG < sizeof(double)) /* assume len(sval) is exactly 1 or 2 */ rpy_reverse_db_emit(sval[1]); } + +static inline void rpy_reverse_db_emit_two_longs(long long value) +{ + Signed sval[2]; + assert(SIZEOF_LONG * 2 == SIZEOF_LONG_LONG); + memcpy(sval, &value, sizeof(value)); + rpy_reverse_db_emit(sval[0]); + rpy_reverse_db_emit(sval[1]); +} diff --git a/rpython/translator/reversedb/test/test_basic.py b/rpython/translator/reversedb/test/test_basic.py --- a/rpython/translator/reversedb/test/test_basic.py +++ b/rpython/translator/reversedb/test/test_basic.py @@ -1,7 +1,34 @@ import py +import os +import array, struct +from rpython.tool.udir import udir from rpython.translator.interactive import Translation +class RDB(object): + def __init__(self, filename): + f = open(filename, 'rb') + f.seek(0, 2) + filesize = f.tell() + f.seek(0, 0) + self.items = array.array("l") + self.items.fromfile(f, filesize / struct.calcsize("l")) + f.close() + # + assert self.items[0] == 0x0A424452 + assert self.items[1] == 0x00FF0001 + assert self.items[2] == 0 + assert self.items[3] == 0 + self.argc = self.items[4] + self.argv = self.items[5] + self.cur = 6 + + def next(self): + n = self.cur + self.cur = n + 1 + return self.items[n] + + class TestBasic(object): def getcompiled(self, entry_point, argtypes, backendopt=True): @@ -14,16 +41,42 @@ t.rtype() if t.backendopt: t.backendopt() - t.compile_c() + self.exename = t.compile_c() + self.rdbname = os.path.join(os.path.dirname(str(self.exename)), + 'log.rdb') def run(*argv): - stdout = t.driver.cbuilder.cmdexec(' '.join(argv)) + env = os.environ.copy() + env['PYPYRDB'] = self.rdbname + stdout = t.driver.cbuilder.cmdexec(' '.join(argv), env=env) return stdout return run + def fetch_rdb(self): + return RDB(self.rdbname) + def test_simple(self): def main(argv): print argv[1:] return 0 fn = self.getcompiled(main, [], backendopt=False) assert fn('abc d') == '[abc, d]\n' + rdb = self.fetch_rdb() + assert rdb.argc == 3 + # + got = [] + for i in range(3): + rdb.next() # ignore the address of argv[i] + s = [] + while True: + c = rdb.next() + if c == 0: + break + s.append(chr(c)) + for c1 in s: + c2 = rdb.next() + assert c2 == ord(c1) + got.append(''.join(s)) + assert rdb.cur == len(rdb.items) + # + assert got == [self.exename, 'abc', 'd'] From pypy.commits at gmail.com Sun Jun 5 11:49:19 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 08:49:19 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Add comments Message-ID: <575449ff.e505c20a.6fe85.ffff828a@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84940:f90fd868636c Date: 2016-06-05 17:50 +0200 http://bitbucket.org/pypy/pypy/changeset/f90fd868636c/ Log: Add comments diff --git a/rpython/translator/reversedb/test/test_basic.py b/rpython/translator/reversedb/test/test_basic.py --- a/rpython/translator/reversedb/test/test_basic.py +++ b/rpython/translator/reversedb/test/test_basic.py @@ -66,17 +66,21 @@ # got = [] for i in range(3): - rdb.next() # ignore the address of argv[i] + rdb.next() # this is from "p = argv[i]" s = [] + # first we determine the length of the "char *p" while True: c = rdb.next() if c == 0: break s.append(chr(c)) + # then we really read the "char *" and copy it into a rpy string + # (that's why this time we don't read the final \0) for c1 in s: c2 = rdb.next() assert c2 == ord(c1) got.append(''.join(s)) + # that's all that should get from this simple example assert rdb.cur == len(rdb.items) # assert got == [self.exename, 'abc', 'd'] From pypy.commits at gmail.com Sun Jun 5 12:42:43 2016 From: pypy.commits at gmail.com (rlamy) Date: Sun, 05 Jun 2016 09:42:43 -0700 (PDT) Subject: [pypy-commit] pypy default: Compatibility with pytest 2.9.* Message-ID: <57545683.665ec20a.79856.ffff935d@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84941:707dee8f0d54 Date: 2016-06-05 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/707dee8f0d54/ Log: Compatibility with pytest 2.9.* diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -5,9 +5,9 @@ from pypy.interpreter.error import OperationError, oefmt try: + from _pytest.assertion.reinterpret import reinterpret as interpret +except ImportError: from _pytest.assertion.newinterpret import interpret -except ImportError: - from _pytest.assertion.oldinterpret import interpret # ____________________________________________________________ From pypy.commits at gmail.com Sun Jun 5 15:57:14 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:14 -0700 (PDT) Subject: [pypy-commit] pypy default: Review all commits to default since PyPy2.7-5.1 Message-ID: <5754841a.e7c9c20a.47bae.ffffc61f@mx.google.com> Author: Matti Picus Branch: Changeset: r84942:873218a739f1 Date: 2016-06-05 22:25 +0300 http://bitbucket.org/pypy/pypy/changeset/873218a739f1/ Log: Review all commits to default since PyPy2.7-5.1 diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -47,7 +47,7 @@ This release supports: * **x86** machines on most common operating systems - (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, @@ -72,7 +72,7 @@ - add prelminary support for PyDateTime_* - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, - - PyAnySet_CheckExact, PyUnicode_Concat + PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo - improve support for PyGILState_Ensure, PyGILState_Release, and thread primitives, also find a case where CPython will allow thread creation before PyEval_InitThreads is run, dissallow on PyPy @@ -80,6 +80,10 @@ - rewrite slot assignment for typeobjects - improve tracking of PyObject to rpython object mapping - support tp_as_{number, sequence, mapping, buffer} slots + - support ByteArrayObject via the new resizable_list_supporting_raw_ptr + - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's + - fix the signature of PyUFunc_FromFuncAndDataAndSignature + - implement many PyWhatever_FOO() as a macro taking a `void *` * CPyExt tweak: instead of "GIL not held when a CPython C extension module calls PyXxx", we now silently acquire/release the GIL. Helps with @@ -93,8 +97,44 @@ * Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap + * Support command line -v to trace import statements + + * Add rposix functions for PyPy3.3 support + + * Give super an __init__ and a simple __new__ for CPython compatibility + + * Revive traceviewer, a tool to use pygame to view traces + + * Update to cffi/847bbc0297f8 which improves help() on cffi objects + * Bug Fixes + * Fix issue #2277: only special-case two exact lists in zip(), not list + subclasses, because an overridden __iter__() should be called (probably) + + * Fix issue #2226: Another tweak in the incremental GC- this should ensure + that progress in the major GC occurs quickly enough in all cases. + + * Clarify and refactor documentation on http://doc.pypy.org + + * Use "must be unicode, not %T" in unicodedata TypeErrors. + + * Manually reset sys.settrace() and sys.setprofile() when we're done running. + This is not exactly what CPython does, but if we get an exception, unlike + CPython, we call functions from the 'traceback' module, and these would + call more the trace/profile function. That's unexpected and can lead + to more crashes at this point. + + * Use the appropriate tp_dealloc on a subclass of a builtin type, and call + tp_new for a python-sublcass of a C-API type + + * Fix for issue #2285 - rare vmprof segfaults on OS/X + + * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc + + * Fix issue #2311 - grab the `__future__` flags imported in the main script, in + `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console + * Issues reported with our previous release were resolved_ after reports from users on our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy @@ -103,6 +143,9 @@ * Implement ufunc.outer on numpypy + * Move PyPy-specific numpy headers to a subdirectory (also changed pypy/numpy + accordingly) + * Performance improvements: * Use bitstrings to compress lists of descriptors that are attached to an @@ -114,11 +157,20 @@ can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly negative. + * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())` + + * Use the __builtin_add_overflow built-ins if they are available + + * Rework the way registers are moved/spilled in before_call() * Internal refactorings: + * Refactor code to better support Python3-compatible syntax + + * Document and refactor OperationError -> oefmt + * Reduce the size of generated C sources during translation by - refactoring function declarations + eliminating many many unused struct declarations (Issue #2281) * Remove a number of translation-time options that were not tested and never used. Also fix a performance bug in the method cache @@ -126,6 +178,9 @@ * Reduce the size of generated code by using the same function objects in all generated subclasses + * Share cpyext Py* function wrappers according to the signature, shrining the + translated libpypy.so by about + * Compile c snippets with -Werror, and fix warnings it exposed .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html From pypy.commits at gmail.com Sun Jun 5 15:57:19 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:19 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: merge default into branch Message-ID: <5754841f.aaf0c20a.96323.ffffdc7d@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84943:b849a6c35c3f Date: 2016-06-05 22:29 +0300 http://bitbucket.org/pypy/pypy/changeset/b849a6c35c3f/ Log: merge default into branch diff too long, truncating to 2000 out of 47348 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,8 @@ 5f8302b8bf9f53056e40426f10c72151564e5b19 release-4.0.1 246c9cf22037b11dc0e8c29ce3f291d3b8c5935a release-5.0 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1 +3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1 +b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 diff --git a/dotviewer/graphserver.py b/dotviewer/graphserver.py --- a/dotviewer/graphserver.py +++ b/dotviewer/graphserver.py @@ -143,6 +143,11 @@ if __name__ == '__main__': if len(sys.argv) != 2: + if len(sys.argv) == 1: + # start locally + import sshgraphserver + sshgraphserver.ssh_graph_server(['LOCAL']) + sys.exit(0) print >> sys.stderr, __doc__ sys.exit(2) if sys.argv[1] == '--stdio': diff --git a/dotviewer/sshgraphserver.py b/dotviewer/sshgraphserver.py --- a/dotviewer/sshgraphserver.py +++ b/dotviewer/sshgraphserver.py @@ -4,11 +4,14 @@ Usage: sshgraphserver.py hostname [more args for ssh...] + sshgraphserver.py LOCAL This logs in to 'hostname' by passing the arguments on the command-line to ssh. No further configuration is required: it works for all programs using the dotviewer library as long as they run on 'hostname' under the same username as the one sshgraphserver logs as. + +If 'hostname' is the string 'LOCAL', then it starts locally without ssh. """ import graphserver, socket, subprocess, random @@ -18,12 +21,19 @@ s1 = socket.socket() s1.bind(('127.0.0.1', socket.INADDR_ANY)) localhost, localport = s1.getsockname() - remoteport = random.randrange(10000, 20000) - # ^^^ and just hope there is no conflict - args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)] - args = args + sshargs + ['python -u -c "exec input()"'] - print ' '.join(args[:-1]) + if sshargs[0] != 'LOCAL': + remoteport = random.randrange(10000, 20000) + # ^^^ and just hope there is no conflict + + args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % ( + remoteport, localport)] + args = args + sshargs + ['python -u -c "exec input()"'] + else: + remoteport = localport + args = ['python', '-u', '-c', 'exec input()'] + + print ' '.join(args) p = subprocess.Popen(args, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE) diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py --- a/lib-python/2.7/distutils/cmd.py +++ b/lib-python/2.7/distutils/cmd.py @@ -298,8 +298,16 @@ src_cmd_obj.ensure_finalized() for (src_option, dst_option) in option_pairs: if getattr(self, dst_option) is None: - setattr(self, dst_option, - getattr(src_cmd_obj, src_option)) + try: + setattr(self, dst_option, + getattr(src_cmd_obj, src_option)) + except AttributeError: + # This was added after problems with setuptools 18.4. + # It seems that setuptools 20.9 fixes the problem. + # But e.g. on Ubuntu 14.04 with /usr/bin/virtualenv + # if I say "virtualenv -p pypy venv-pypy" then it + # just installs setuptools 18.4 from some cache... + pass def get_finalized_command(self, command, create=1): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -834,54 +834,63 @@ c2pread, c2pwrite = None, None errread, errwrite = None, None + ispread = False if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = _subprocess.CreatePipe(None, 0) + ispread = True elif stdin == PIPE: p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) + ispread = True elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) + p2cread = self._make_inheritable(p2cread, ispread) # We just duplicated the handle, it has to be closed at the end to_close.add(p2cread) if stdin == PIPE: to_close.add(p2cwrite) + ispwrite = False if stdout is None: c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stdout == PIPE: c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) + c2pwrite = self._make_inheritable(c2pwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(c2pwrite) if stdout == PIPE: to_close.add(c2pread) + ispwrite = False if stderr is None: errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) if errwrite is None: _, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == PIPE: errread, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == STDOUT: - errwrite = c2pwrite.handle # pass id to not close it + errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) + errwrite = self._make_inheritable(errwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(errwrite) if stderr == PIPE: @@ -892,13 +901,14 @@ errread, errwrite), to_close - def _make_inheritable(self, handle): + def _make_inheritable(self, handle, close=False): """Return a duplicate of handle, which is inheritable""" dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), handle, _subprocess.GetCurrentProcess(), 0, 1, _subprocess.DUPLICATE_SAME_ACCESS) - # If the initial handle was obtained with CreatePipe, close it. - if not isinstance(handle, int): + # PyPy: If the initial handle was obtained with CreatePipe, + # close it. + if close: handle.Close() return dupl diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py --- a/lib-python/2.7/test/test_descr.py +++ b/lib-python/2.7/test/test_descr.py @@ -1735,7 +1735,6 @@ ("__reversed__", reversed, empty_seq, set(), {}), ("__length_hint__", list, zero, set(), {"__iter__" : iden, "next" : stop}), - ("__sizeof__", sys.getsizeof, zero, set(), {}), ("__instancecheck__", do_isinstance, return_true, set(), {}), ("__missing__", do_dict_missing, some_number, set(("__class__",)), {}), @@ -1747,6 +1746,8 @@ ("__format__", format, format_impl, set(), {}), ("__dir__", dir, empty_seq, set(), {}), ] + if test_support.check_impl_detail(): + specials.append(("__sizeof__", sys.getsizeof, zero, set(), {})) class Checker(object): def __getattr__(self, attr, test=self): @@ -1768,10 +1769,6 @@ raise MyException for name, runner, meth_impl, ok, env in specials: - if name == '__length_hint__' or name == '__sizeof__': - if not test_support.check_impl_detail(): - continue - class X(Checker): pass for attr, obj in env.iteritems(): diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py --- a/lib-python/2.7/test/test_sys_settrace.py +++ b/lib-python/2.7/test/test_sys_settrace.py @@ -328,8 +328,8 @@ def test_13_genexp(self): if self.using_gc: + gc.enable() test_support.gc_collect() - gc.enable() try: self.run_test(generator_example) # issue1265: if the trace function contains a generator, diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt --- a/lib-python/stdlib-upgrade.txt +++ b/lib-python/stdlib-upgrade.txt @@ -5,15 +5,23 @@ overly detailed -1. check out the branch vendor/stdlib +0. make sure your working dir is clean +1. check out the branch vendor/stdlib (for 2.7) or vendor/stdlib-3-* (for py3k) + or create branch vendor/stdlib-3-* 2. upgrade the files there + 2a. remove lib-python/2.7/ or lib-python/3/ + 2b. copy the files from the cpython repo + 2c. hg add lib-python/2.7/ or lib-python/3/ + 2d. hg remove --after + 2e. show copied files in cpython repo by running `hg diff --git -r v -r v Lib | grep '^copy \(from\|to\)'` + 2f. fix copies / renames manually by running `hg copy --after ` for each copied file 3. update stdlib-version.txt with the output of hg -id from the cpython repo 4. commit -5. update to default/py3k +5. update to default / py3k 6. create a integration branch for the new stdlib (just hg branch stdlib-$version) -7. merge vendor/stdlib +7. merge vendor/stdlib or vendor/stdlib-3-* 8. commit 10. fix issues 11. commit --close-branch -12. merge to default +12. merge to default / py3k diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py --- a/lib_pypy/_pypy_interact.py +++ b/lib_pypy/_pypy_interact.py @@ -6,7 +6,7 @@ irc_header = "And now for something completely different" -def interactive_console(mainmodule=None, quiet=False): +def interactive_console(mainmodule=None, quiet=False, future_flags=0): # set sys.{ps1,ps2} just before invoking the interactive interpreter. This # mimics what CPython does in pythonrun.c if not hasattr(sys, 'ps1'): @@ -37,15 +37,17 @@ raise ImportError from pyrepl.simple_interact import run_multiline_interactive_console except ImportError: - run_simple_interactive_console(mainmodule) + run_simple_interactive_console(mainmodule, future_flags=future_flags) else: - run_multiline_interactive_console(mainmodule) + run_multiline_interactive_console(mainmodule, future_flags=future_flags) -def run_simple_interactive_console(mainmodule): +def run_simple_interactive_console(mainmodule, future_flags=0): import code if mainmodule is None: import __main__ as mainmodule console = code.InteractiveConsole(mainmodule.__dict__, filename='') + if future_flags: + console.compile.compiler.flags |= future_flags # some parts of code.py are copied here because it seems to be impossible # to start an interactive console without printing at least one line # of banner diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py --- a/lib_pypy/_pypy_irc_topic.py +++ b/lib_pypy/_pypy_irc_topic.py @@ -224,23 +224,9 @@ va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat """ -from string import ascii_uppercase, ascii_lowercase - def rot13(data): - """ A simple rot-13 encoder since `str.encode('rot13')` was removed from - Python as of version 3.0. It rotates both uppercase and lowercase letters individually. - """ - total = [] - for char in data: - if char in ascii_uppercase: - index = (ascii_uppercase.find(char) + 13) % 26 - total.append(ascii_uppercase[index]) - elif char in ascii_lowercase: - index = (ascii_lowercase.find(char) + 13) % 26 - total.append(ascii_lowercase[index]) - else: - total.append(char) - return "".join(total) + return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else + -13 if 'N'<=c.upper()<='Z' else 0)) for c in data) def some_topic(): import time diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py --- a/lib_pypy/_subprocess.py +++ b/lib_pypy/_subprocess.py @@ -4,6 +4,9 @@ subprocess module on Windows. """ +import sys +if sys.platform != 'win32': + raise ImportError("The '_subprocess' module is only available on Windows") # Declare external Win32 functions diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ @@ -397,20 +397,7 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gcp = self._backend.gcp - except AttributeError: - pass - else: - return gcp(cdata, destructor) - # - with self._lock: - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + return self._backend.gcp(cdata, destructor) def _get_cached_btype(self, type): assert self._lock.acquire(False) is False diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -460,6 +460,11 @@ return x._value raise TypeError("character expected, got %s" % type(x).__name__) + def __nonzero__(self): + return ord(self._value) != 0 + else: + def __nonzero__(self): + return self._value != 0 if kind == 'float': @staticmethod @@ -993,6 +998,31 @@ assert onerror is None # XXX not implemented return BType(source, error) + def gcp(self, cdata, destructor): + BType = self.typeof(cdata) + + if destructor is None: + if not (hasattr(BType, '_gcp_type') and + BType._gcp_type is BType): + raise TypeError("Can remove destructor only on a object " + "previously returned by ffi.gc()") + cdata._destructor = None + return None + + try: + gcp_type = BType._gcp_type + except AttributeError: + class CTypesDataGcp(BType): + __slots__ = ['_orig', '_destructor'] + def __del__(self): + if self._destructor is not None: + self._destructor(self._orig) + gcp_type = BType._gcp_type = CTypesDataGcp + new_cdata = self.cast(gcp_type, cdata) + new_cdata._orig = cdata + new_cdata._destructor = destructor + return new_cdata + typeof = type def getcname(self, BType, replace_with): diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py --- a/lib_pypy/cffi/commontypes.py +++ b/lib_pypy/cffi/commontypes.py @@ -35,8 +35,11 @@ "you call ffi.set_unicode()" % (commontype,)) else: if commontype == cdecl: - raise api.FFIError("Unsupported type: %r. Please file a bug " - "if you think it should be." % (commontype,)) + raise api.FFIError( + "Unsupported type: %r. Please look at " + "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " + "and file an issue if you think this type should really " + "be supported." % (commontype,)) result, quals = parser.parse_type_and_quals(cdecl) # recursive assert isinstance(result, model.BaseTypeByIdentity) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -814,7 +814,7 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) << 1); /* check that '%s.%s' is " + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' @@ -991,7 +991,7 @@ prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') prnt(' int n = (%s) <= 0;' % (name,)) - prnt(' *o = (unsigned long long)((%s) << 0);' + prnt(' *o = (unsigned long long)((%s) | 0);' ' /* check that %s is an integer */' % (name, name)) if check_value is not None: if check_value > 0: @@ -1250,7 +1250,7 @@ def _emit_bytecode_UnknownIntegerType(self, tp, index): s = ('_cffi_prim_int(sizeof(%s), (\n' - ' ((%s)-1) << 0 /* check that %s is an integer type */\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' ' ) <= 0)' % (tp.name, tp.name, tp.name)) self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py --- a/lib_pypy/pyrepl/simple_interact.py +++ b/lib_pypy/pyrepl/simple_interact.py @@ -43,11 +43,13 @@ return short return text -def run_multiline_interactive_console(mainmodule=None): +def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code if mainmodule is None: import __main__ as mainmodule console = code.InteractiveConsole(mainmodule.__dict__, filename='') + if future_flags: + console.compile.compiler.flags |= future_flags def more_lines(unicodetext): # ooh, look at the hack: diff --git a/lib_pypy/syslog.py b/lib_pypy/syslog.py --- a/lib_pypy/syslog.py +++ b/lib_pypy/syslog.py @@ -51,6 +51,8 @@ # if log is not opened, open it now if not _S_log_open: openlog() + if isinstance(message, unicode): + message = str(message) lib.syslog(priority, "%s", message) @builtinify diff --git a/pypy/__init__.py b/pypy/__init__.py --- a/pypy/__init__.py +++ b/pypy/__init__.py @@ -1,4 +1,5 @@ -# Empty +import os +pypydir = os.path.realpath(os.path.dirname(__file__)) # XXX Should be empty again, soon. # XXX hack for win64: diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -204,15 +204,6 @@ BoolOption("withstrbuf", "use strings optimized for addition (ver 2)", default=False), - BoolOption("withprebuiltchar", - "use prebuilt single-character string objects", - default=False), - - BoolOption("sharesmallstr", - "always reuse the prebuilt string objects " - "(the empty string and potentially single-char strings)", - default=False), - BoolOption("withspecialisedtuple", "use specialised tuples", default=False), @@ -222,39 +213,14 @@ default=False, requires=[("objspace.honor__builtins__", False)]), - BoolOption("withmapdict", - "make instances really small but slow without the JIT", - default=False, - requires=[("objspace.std.getattributeshortcut", True), - ("objspace.std.withtypeversion", True), - ]), - - BoolOption("withrangelist", - "enable special range list implementation that does not " - "actually create the full list until the resulting " - "list is mutated", - default=False), BoolOption("withliststrategies", "enable optimized ways to store lists of primitives ", default=True), - BoolOption("withtypeversion", - "version type objects when changing them", - cmdline=None, - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), - - BoolOption("withmethodcache", - "try to cache method lookups", - default=False, - requires=[("objspace.std.withtypeversion", True), - ("translation.rweakref", True)]), BoolOption("withmethodcachecounter", "try to cache methods and provide a counter in __pypy__. " "for testing purposes only.", - default=False, - requires=[("objspace.std.withmethodcache", True)]), + default=False), IntOption("methodcachesizeexp", " 2 ** methodcachesizeexp is the size of the of the method cache ", default=11), @@ -265,22 +231,10 @@ BoolOption("optimized_list_getitem", "special case the 'list[integer]' expressions", default=False), - BoolOption("getattributeshortcut", - "track types that override __getattribute__", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), BoolOption("newshortcut", "cache and shortcut calling __new__ from builtin types", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), + default=False), - BoolOption("withidentitydict", - "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), ]), ]) @@ -296,15 +250,10 @@ """ # all the good optimizations for PyPy should be listed here if level in ['2', '3', 'jit']: - config.objspace.std.suggest(withrangelist=True) - config.objspace.std.suggest(withmethodcache=True) - config.objspace.std.suggest(withprebuiltchar=True) config.objspace.std.suggest(intshortcut=True) config.objspace.std.suggest(optimized_list_getitem=True) - config.objspace.std.suggest(getattributeshortcut=True) #config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withspecialisedtuple=True) - config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) @@ -317,16 +266,13 @@ # memory-saving optimizations if level == 'mem': config.objspace.std.suggest(withprebuiltint=True) - config.objspace.std.suggest(withrangelist=True) - config.objspace.std.suggest(withprebuiltchar=True) - config.objspace.std.suggest(withmapdict=True) + config.objspace.std.suggest(withliststrategies=True) if not IS_64_BITS: config.objspace.std.suggest(withsmalllong=True) # extra optimizations with the JIT if level == 'jit': config.objspace.std.suggest(withcelldict=True) - config.objspace.std.suggest(withmapdict=True) def enable_allworkingmodules(config): diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py --- a/pypy/config/test/test_pypyoption.py +++ b/pypy/config/test/test_pypyoption.py @@ -11,12 +11,6 @@ assert conf.objspace.usemodules.gc - conf.objspace.std.withmapdict = True - assert conf.objspace.std.withtypeversion - conf = get_pypy_config() - conf.objspace.std.withtypeversion = False - py.test.raises(ConfigError, "conf.objspace.std.withmapdict = True") - def test_conflicting_gcrootfinder(): conf = get_pypy_config() conf.translation.gc = "boehm" @@ -47,18 +41,10 @@ def test_set_pypy_opt_level(): conf = get_pypy_config() set_pypy_opt_level(conf, '2') - assert conf.objspace.std.getattributeshortcut + assert conf.objspace.std.intshortcut conf = get_pypy_config() set_pypy_opt_level(conf, '0') - assert not conf.objspace.std.getattributeshortcut - -def test_rweakref_required(): - conf = get_pypy_config() - conf.translation.rweakref = False - set_pypy_opt_level(conf, '3') - - assert not conf.objspace.std.withtypeversion - assert not conf.objspace.std.withmethodcache + assert not conf.objspace.std.intshortcut def test_check_documentation(): def check_file_exists(fn): diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -1,4 +1,4 @@ -import py, pytest, sys, os, textwrap +import py, pytest, sys, textwrap from inspect import isclass # pytest settings @@ -10,8 +10,6 @@ # option = None -pypydir = os.path.realpath(os.path.dirname(__file__)) - def braindead_deindent(self): """monkeypatch that wont end up doing stupid in the python tokenizer""" text = '\n'.join(self.lines) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -70,9 +70,6 @@ bz2 libbz2 -lzma (PyPy3 only) - liblzma - pyexpat libexpat1 @@ -98,19 +95,24 @@ tk tk-dev +lzma (PyPy3 only) + liblzma + +To run untranslated tests, you need the Boehm garbage collector libgc. + On Debian, this is the command to install all build-time dependencies:: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ - tk-dev + tk-dev libgc-dev liblzma-dev For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. On Fedora:: - yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ - lib-sqlite3-devel ncurses-devel expat-devel openssl-devel - (XXX plus the Febora version of libgdbm-dev and tk-dev) + dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ + lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \ + gdbm-devel For the optional lzma module on PyPy3 you will also need ``xz-devel``. diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -266,7 +266,13 @@ To raise an application-level exception:: - raise OperationError(space.w_XxxError, space.wrap("message")) + from pypy.interpreter.error import oefmt + + raise oefmt(space.w_XxxError, "message") + + raise oefmt(space.w_XxxError, "file '%s' not found in '%s'", filename, dir) + + raise oefmt(space.w_XxxError, "file descriptor '%d' not open", fd) To catch a specific application-level exception:: diff --git a/pypy/doc/config/objspace.std.getattributeshortcut.txt b/pypy/doc/config/objspace.std.getattributeshortcut.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.getattributeshortcut.txt +++ /dev/null @@ -1,1 +0,0 @@ -Performance only: track types that override __getattribute__. diff --git a/pypy/doc/config/objspace.std.methodcachesizeexp.txt b/pypy/doc/config/objspace.std.methodcachesizeexp.txt --- a/pypy/doc/config/objspace.std.methodcachesizeexp.txt +++ b/pypy/doc/config/objspace.std.methodcachesizeexp.txt @@ -1,1 +1,1 @@ -Set the cache size (number of entries) for :config:`objspace.std.withmethodcache`. +Set the cache size (number of entries) for the method cache. diff --git a/pypy/doc/config/objspace.std.withidentitydict.txt b/pypy/doc/config/objspace.std.withidentitydict.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withidentitydict.txt +++ /dev/null @@ -1,21 +0,0 @@ -============================= -objspace.std.withidentitydict -============================= - -* **name:** withidentitydict - -* **description:** enable a dictionary strategy for "by identity" comparisons - -* **command-line:** --objspace-std-withidentitydict - -* **command-line for negation:** --no-objspace-std-withidentitydict - -* **option type:** boolean option - -* **default:** True - - -Enable a dictionary strategy specialized for instances of classes which -compares "by identity", which is the default unless you override ``__hash__``, -``__eq__`` or ``__cmp__``. This strategy will be used only with new-style -classes. diff --git a/pypy/doc/config/objspace.std.withmapdict.txt b/pypy/doc/config/objspace.std.withmapdict.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withmapdict.txt +++ /dev/null @@ -1,5 +0,0 @@ -Enable the new version of "sharing dictionaries". - -See the section in `Standard Interpreter Optimizations`_ for more details. - -.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#sharing-dicts diff --git a/pypy/doc/config/objspace.std.withmethodcache.txt b/pypy/doc/config/objspace.std.withmethodcache.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withmethodcache.txt +++ /dev/null @@ -1,2 +0,0 @@ -Enable method caching. See the section "Method Caching" in `Standard -Interpreter Optimizations <../interpreter-optimizations.html#method-caching>`__. diff --git a/pypy/doc/config/objspace.std.withmethodcachecounter.txt b/pypy/doc/config/objspace.std.withmethodcachecounter.txt --- a/pypy/doc/config/objspace.std.withmethodcachecounter.txt +++ b/pypy/doc/config/objspace.std.withmethodcachecounter.txt @@ -1,1 +1,1 @@ -Testing/debug option for :config:`objspace.std.withmethodcache`. +Testing/debug option for the method cache. diff --git a/pypy/doc/config/objspace.std.withprebuiltchar.txt b/pypy/doc/config/objspace.std.withprebuiltchar.txt deleted file mode 100644 diff --git a/pypy/doc/config/objspace.std.withrangelist.txt b/pypy/doc/config/objspace.std.withrangelist.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withrangelist.txt +++ /dev/null @@ -1,11 +0,0 @@ -Enable "range list" objects. They are an additional implementation of the Python -``list`` type, indistinguishable for the normal user. Whenever the ``range`` -builtin is called, an range list is returned. As long as this list is not -mutated (and for example only iterated over), it uses only enough memory to -store the start, stop and step of the range. This makes using ``range`` as -efficient as ``xrange``, as long as the result is only used in a ``for``-loop. - -See the section in `Standard Interpreter Optimizations`_ for more details. - -.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#range-lists - diff --git a/pypy/doc/config/objspace.std.withtypeversion.txt b/pypy/doc/config/objspace.std.withtypeversion.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withtypeversion.txt +++ /dev/null @@ -1,6 +0,0 @@ -This (mostly internal) option enables "type versions": Every type object gets an -(only internally visible) version that is updated when the type's dict is -changed. This is e.g. used for invalidating caches. It does not make sense to -enable this option alone. - -.. internal diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -12,9 +12,9 @@ The work on the cling backend has so far been done only for CPython, but bringing it to PyPy is a lot less work than developing it in the first place. -.. _Reflex: http://root.cern.ch/drupal/content/reflex -.. _CINT: http://root.cern.ch/drupal/content/cint -.. _cling: http://root.cern.ch/drupal/content/cling +.. _Reflex: https://root.cern.ch/how/how-use-reflex +.. _CINT: https://root.cern.ch/introduction-cint +.. _cling: https://root.cern.ch/cling .. _llvm: http://llvm.org/ .. _clang: http://clang.llvm.org/ @@ -283,7 +283,8 @@ core reflection set, but for the moment assume we want to have it in the reflection library that we are building for this example. -The ``genreflex`` script can be steered using a so-called `selection file`_, +The ``genreflex`` script can be steered using a so-called `selection file`_ +(see "Generating Reflex Dictionaries") which is a simple XML file specifying, either explicitly or by using a pattern, which classes, variables, namespaces, etc. to select from the given header file. @@ -305,7 +306,7 @@ -.. _selection file: http://root.cern.ch/drupal/content/generating-reflex-dictionaries +.. _selection file: https://root.cern.ch/how/how-use-reflex Now the reflection info can be generated and compiled:: @@ -811,7 +812,7 @@ immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment variable. -.. _PyROOT: http://root.cern.ch/drupal/content/pyroot +.. _PyROOT: https://root.cern.ch/pyroot There are a couple of minor differences between PyCintex and cppyy, most to do with naming. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -387,6 +387,14 @@ wrappers. On PyPy we can't tell the difference, so ``ismethod([].__add__) == ismethod(list.__add__) == True``. +* in CPython, the built-in types have attributes that can be + implemented in various ways. Depending on the way, if you try to + write to (or delete) a read-only (or undeletable) attribute, you get + either a ``TypeError`` or an ``AttributeError``. PyPy tries to + strike some middle ground between full consistency and full + compatibility here. This means that a few corner cases don't raise + the same exception, like ``del (lambda:None).__closure__``. + * in pure Python, if you write ``class A(object): def f(self): pass`` and have a subclass ``B`` which doesn't override ``f()``, then ``B.f(x)`` still checks that ``x`` is an instance of ``B``. In diff --git a/pypy/doc/dir-reference.rst b/pypy/doc/dir-reference.rst --- a/pypy/doc/dir-reference.rst +++ b/pypy/doc/dir-reference.rst @@ -21,7 +21,7 @@ :source:`pypy/doc/discussion/` drafts of ideas and documentation -:source:`pypy/goal/` our :ref:`main PyPy-translation scripts ` +:source:`pypy/goal/` our main PyPy-translation scripts live here :source:`pypy/interpreter/` :doc:`bytecode interpreter ` and related objects diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst --- a/pypy/doc/discussion/finalizer-order.rst +++ b/pypy/doc/discussion/finalizer-order.rst @@ -1,19 +1,127 @@ -.. XXX armin, what do we do with this? +Ordering finalizers in the MiniMark GC +====================================== -Ordering finalizers in the SemiSpace GC -======================================= +RPython interface +----------------- -Goal ----- +In RPython programs like PyPy, we need a fine-grained method of +controlling the RPython- as well as the app-level ``__del__()``. To +make it possible, the RPython interface is now the following one (from +May 2016): -After a collection, the SemiSpace GC should call the finalizers on +* RPython objects can have ``__del__()``. These are called + immediately by the GC when the last reference to the object goes + away, like in CPython. However, the long-term goal is that all + ``__del__()`` methods should only contain simple enough code. If + they do, we call them "destructors". They can't use operations that + would resurrect the object, for example. Use the decorator + ``@rgc.must_be_light_finalizer`` to ensure they are destructors. + +* RPython-level ``__del__()`` that are not passing the destructor test + are supported for backward compatibility, but deprecated. The rest + of this document assumes that ``__del__()`` are all destructors. + +* For any more advanced usage --- in particular for any app-level + object with a __del__ --- we don't use the RPython-level + ``__del__()`` method. Instead we use + ``rgc.FinalizerController.register_finalizer()``. This allows us to + attach a finalizer method to the object, giving more control over + the ordering than just an RPython ``__del__()``. + +We try to consistently call ``__del__()`` a destructor, to distinguish +it from a finalizer. A finalizer runs earlier, and in topological +order; care must be taken that the object might still be reachable at +this point if we're clever enough. A destructor on the other hand runs +last; nothing can be done with the object any more, and the GC frees it +immediately. + + +Destructors +----------- + +A destructor is an RPython ``__del__()`` method that is called directly +by the GC when it is about to free the memory. Intended for objects +that just need to free an extra block of raw memory. + +There are restrictions on the kind of code you can put in ``__del__()``, +including all other functions called by it. These restrictions are +checked. In particular you cannot access fields containing GC objects. +Right now you can't call any external C function either. + +Destructors are called precisely when the GC frees the memory of the +object. As long as the object exists (even in some finalizer queue or +anywhere), its destructor is not called. + + +Register_finalizer +------------------ + +The interface for full finalizers is made with PyPy in mind, but should +be generally useful. + +The idea is that you subclass the ``rgc.FinalizerQueue`` class:: + +* You must give a class-level attribute ``base_class``, which is the + base class of all instances with a finalizer. (If you need + finalizers on several unrelated classes, you need several unrelated + ``FinalizerQueue`` subclasses.) + +* You override the ``finalizer_trigger()`` method; see below. + +Then you create one global (or space-specific) instance of this +subclass; call it ``fin``. At runtime, you call +``fin.register_finalizer(obj)`` for every instance ``obj`` that needs +a finalizer. Each ``obj`` must be an instance of ``fin.base_class``, +but not every such instance needs to have a finalizer registered; +typically we try to register a finalizer on as few objects as possible +(e.g. only if it is an object which has an app-level ``__del__()`` +method). + +After a major collection, the GC finds all objects ``obj`` on which a +finalizer was registered and which are unreachable, and mark them as +reachable again, as well as all objects they depend on. It then picks +a topological ordering (breaking cycles randomly, if any) and enqueues +the objects and their registered finalizer functions in that order, in +a queue specific to the prebuilt ``fin`` instance. Finally, when the +major collection is done, it calls ``fin.finalizer_trigger()``. + +This method ``finalizer_trigger()`` can either do some work directly, +or delay it to be done later (e.g. between two bytecodes). If it does +work directly, note that it cannot (directly or indirectly) cause the +GIL to be released. + +To find the queued items, call ``fin.next_dead()`` repeatedly. It +returns the next queued item, or ``None`` when the queue is empty. + +In theory, it would kind of work if you cumulate several different +``FinalizerQueue`` instances for objects of the same class, and +(always in theory) the same ``obj`` could be registered several times +in the same queue, or in several queues. This is not tested though. +For now the untranslated emulation does not support registering the +same object several times. + +Note that the Boehm garbage collector, used in ``rpython -O0``, +completely ignores ``register_finalizer()``. + + +Ordering of finalizers +---------------------- + +After a collection, the MiniMark GC should call the finalizers on *some* of the objects that have one and that have become unreachable. Basically, if there is a reference chain from an object a to an object b then it should not call the finalizer for b immediately, but just keep b alive and try again to call its finalizer after the next collection. -This basic idea fails when there are cycles. It's not a good idea to +(Note that this creates rare but annoying issues as soon as the program +creates chains of objects with finalizers more quickly than the rate at +which major collections go (which is very slow). In August 2013 we tried +instead to call all finalizers of all objects found unreachable at a major +collection. That branch, ``gc-del``, was never merged. It is still +unclear what the real consequences would be on programs in the wild.) + +The basic idea fails in the presence of cycles. It's not a good idea to keep the objects alive forever or to never call any of the finalizers. The model we came up with is that in this case, we could just call the finalizer of one of the objects in the cycle -- but only, of course, if @@ -33,6 +141,7 @@ detach the finalizer (so that it's not called more than once) call the finalizer + Algorithm --------- @@ -136,28 +245,8 @@ that doesn't change the state of an object, we don't follow its children recursively. -In practice, in the SemiSpace, Generation and Hybrid GCs, we can encode -the 4 states with a single extra bit in the header: - - ===== ============= ======== ==================== - state is_forwarded? bit set? bit set in the copy? - ===== ============= ======== ==================== - 0 no no n/a - 1 no yes n/a - 2 yes yes yes - 3 yes whatever no - ===== ============= ======== ==================== - -So the loop above that does the transition from state 1 to state 2 is -really just a copy(x) followed by scan_copied(). We must also clear the -bit in the copy at the end, to clean up before the next collection -(which means recursively bumping the state from 2 to 3 in the final -loop). - -In the MiniMark GC, the objects don't move (apart from when they are -copied out of the nursery), but we use the flag GCFLAG_VISITED to mark -objects that survive, so we can also have a single extra bit for -finalizers: +In practice, in the MiniMark GCs, we can encode +the 4 states with a combination of two bits in the header: ===== ============== ============================ state GCFLAG_VISITED GCFLAG_FINALIZATION_ORDERING @@ -167,3 +256,8 @@ 2 yes yes 3 yes no ===== ============== ============================ + +So the loop above that does the transition from state 1 to state 2 is +really just a recursive visit. We must also clear the +FINALIZATION_ORDERING bit at the end (state 2 to state 3) to clean up +before the next collection. diff --git a/pypy/doc/discussions.rst b/pypy/doc/discussions.rst --- a/pypy/doc/discussions.rst +++ b/pypy/doc/discussions.rst @@ -13,3 +13,4 @@ discussion/improve-rpython discussion/ctypes-implementation discussion/jit-profiler + discussion/rawrefcount diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst --- a/pypy/doc/extending.rst +++ b/pypy/doc/extending.rst @@ -79,7 +79,7 @@ :doc:`Full details ` are `available here `. .. _installed separately: http://cern.ch/wlav/reflex-2013-08-14.tar.bz2 -.. _Reflex: http://root.cern.ch/drupal/content/reflex +.. _Reflex: https://root.cern.ch/how/how-use-reflex RPython Mixed Modules diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -106,20 +106,33 @@ For information on which third party extensions work (or do not work) with PyPy see the `compatibility wiki`_. +For more information about how we manage refcounting semamtics see +rawrefcount_ + .. _compatibility wiki: https://bitbucket.org/pypy/compatibility/wiki/Home .. _cffi: http://cffi.readthedocs.org/ +.. _rawrefcount: discussion/rawrefcount.html On which platforms does PyPy run? --------------------------------- -PyPy is regularly and extensively tested on Linux machines. It mostly +PyPy currently supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +PyPy is regularly and extensively tested on Linux machines. It works on Mac and Windows: it is tested there, but most of us are running -Linux so fixes may depend on 3rd-party contributions. PyPy's JIT -works on x86 (32-bit or 64-bit) and on ARM (ARMv6 or ARMv7). -Support for POWER (64-bit) is stalled at the moment. +Linux so fixes may depend on 3rd-party contributions. -To bootstrap from sources, PyPy can use either CPython (2.6 or 2.7) or +To bootstrap from sources, PyPy can use either CPython 2.7 or another (e.g. older) PyPy. Cross-translation is not really supported: e.g. to build a 32-bit PyPy, you need to have a 32-bit environment. Cross-translation is only explicitly supported between a 32-bit Intel diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-5.1.1.rst release-5.1.0.rst release-5.0.1.rst release-5.0.0.rst @@ -48,6 +49,13 @@ release-0.6 +CPython 3.3 compatible versions +------------------------------- + +.. toctree:: + + release-pypy3.3-v5.2-alpha1.rst + CPython 3.2 compatible versions ------------------------------- diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst --- a/pypy/doc/interpreter-optimizations.rst +++ b/pypy/doc/interpreter-optimizations.rst @@ -62,29 +62,37 @@ Dictionary Optimizations ~~~~~~~~~~~~~~~~~~~~~~~~ -Multi-Dicts -+++++++++++ +Dict Strategies +++++++++++++++++ -Multi-dicts are a special implementation of dictionaries. It became clear that -it is very useful to *change* the internal representation of an object during -its lifetime. Multi-dicts are a general way to do that for dictionaries: they -provide generic support for the switching of internal representations for -dicts. +Dict strategies are an implementation approach for dictionaries (and lists) +that make it possible to use a specialized representation of the dictionary's +data, while still being able to switch back to a general representation should +that become necessary later. -If you just enable multi-dicts, special representations for empty dictionaries, -for string-keyed dictionaries. In addition there are more specialized dictionary -implementations for various purposes (see below). +Dict strategies are always enabled, by default there are special strategies for +dicts with just string keys, just unicode keys and just integer keys. If one of +those specialized strategies is used, then dict lookup can use much faster +hashing and comparison for the dict keys. There is of course also a strategy +for general keys. -This is now the default implementation of dictionaries in the Python interpreter. +Identity Dicts ++++++++++++++++ -Sharing Dicts +We also have a strategy specialized for keys that are instances of classes +which compares "by identity", which is the default unless you override +``__hash__``, ``__eq__`` or ``__cmp__``. This strategy will be used only with +new-style classes. + + +Map Dicts +++++++++++++ -Sharing dictionaries are a special representation used together with multidicts. -This dict representation is used only for instance dictionaries and tries to -make instance dictionaries use less memory (in fact, in the ideal case the -memory behaviour should be mostly like that of using __slots__). +Map dictionaries are a special representation used together with dict strategies. +This dict strategy is used only for instance dictionaries and tries to +make instance dictionaries use less memory (in fact, usually memory behaviour +should be mostly like that of using ``__slots__``). The idea is the following: Most instances of the same class have very similar attributes, and are even adding these keys to the dictionary in the same order @@ -95,8 +103,6 @@ dicts: the representation of the instance dict contains only a list of values. -A more advanced version of sharing dicts, called *map dicts,* is available -with the :config:`objspace.std.withmapdict` option. List Optimizations @@ -114,8 +120,8 @@ created. This gives the memory and speed behaviour of ``xrange`` and the generality of use of ``range``, and makes ``xrange`` essentially useless. -You can enable this feature with the :config:`objspace.std.withrangelist` -option. +This feature is enabled by default as part of the +:config:`objspace.std.withliststrategies` option. User Class Optimizations @@ -133,8 +139,7 @@ base classes is changed). On subsequent lookups the cached version can be used, as long as the instance did not shadow any of its classes attributes. -You can enable this feature with the :config:`objspace.std.withmethodcache` -option. +This feature is enabled by default. Interpreter Optimizations diff --git a/pypy/doc/introduction.rst b/pypy/doc/introduction.rst --- a/pypy/doc/introduction.rst +++ b/pypy/doc/introduction.rst @@ -1,16 +1,22 @@ What is PyPy? ============= -In common parlance, PyPy has been used to mean two things. The first is the -:ref:`RPython translation toolchain `, which is a framework for generating -dynamic programming language implementations. And the second is one -particular implementation that is so generated -- -an implementation of the Python_ programming language written in -Python itself. It is designed to be flexible and easy to experiment with. +Historically, PyPy has been used to mean two things. The first is the +:ref:`RPython translation toolchain ` for generating +interpreters for dynamic programming languages. And the second is one +particular implementation of Python_ produced with it. Because RPython +uses the same syntax as Python, this generated version became known as +Python interpreter written in Python. It is designed to be flexible and +easy to experiment with. -This double usage has proven to be confusing, and we are trying to move -away from using the word PyPy to mean both things. From now on we will -try to use PyPy to only mean the Python implementation, and say the +To make it more clear, we start with source code written in RPython, +apply the RPython translation toolchain, and end up with PyPy as a +binary executable. This executable is the Python interpreter. + +Double usage has proven to be confusing, so we've moved away from using +the word PyPy to mean both toolchain and generated interpreter. Now we +use word PyPy to refer to the Python implementation, and explicitly +mention :ref:`RPython translation toolchain ` when we mean the framework. Some older documents, presentations, papers and videos will still have the old diff --git a/pypy/doc/release-5.1.0.rst b/pypy/doc/release-5.1.0.rst --- a/pypy/doc/release-5.1.0.rst +++ b/pypy/doc/release-5.1.0.rst @@ -3,10 +3,17 @@ ======== We have released PyPy 5.1, about a month after PyPy 5.0. -We encourage all users of PyPy to update to this version. Apart from the usual -bug fixes, there is an ongoing effort to improve the warmup time and memory -usage of JIT-related metadata, and we now fully support the IBM s390x -architecture. + +This release includes more improvement to warmup time and memory +requirements. We have seen about a 20% memory requirement reduction and up to +30% warmup time improvement, more detail in the `blog post`_. + +We also now have `fully support for the IBM s390x`_. Since this support is in +`RPython`_, any dynamic language written using RPython, like PyPy, will +automagically be supported on that architecture. + +We updated cffi_ to 1.6, and continue to improve support for the wider +python ecosystem using the PyPy interpreter. You can download the PyPy 5.1 release here: @@ -26,6 +33,9 @@ .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html .. _`numpy`: https://bitbucket.org/pypy/numpy +.. _cffi: https://cffi.readthedocs.org +.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html +.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html What is PyPy? ============= @@ -46,7 +56,7 @@ * big- and little-endian variants of **PPC64** running Linux, - * **s960x** running Linux + * **s390x** running Linux .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org .. _`dynamic languages`: http://pypyjs.org @@ -74,6 +84,8 @@ * Fix a corner case in the JIT * Fix edge cases in the cpyext refcounting-compatible semantics + (more work on cpyext compatibility is coming in the ``cpyext-ext`` + branch, but isn't ready yet) * Try harder to not emit NEON instructions on ARM processors without NEON support @@ -92,11 +104,17 @@ * Fix sandbox startup (a regression in 5.0) + * Fix possible segfault for classes with mangled mro or __metaclass__ + + * Fix isinstance(deque(), Hashable) on the pure python deque + + * Fix an issue with forkpty() + * Issues reported with our previous release were resolved_ after reports from users on our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy -* Numpy: +* Numpy_: * Implemented numpy.where for a single argument @@ -108,6 +126,8 @@ functions exported from libpypy.so are declared in pypy_numpy.h, which is included only when building our fork of numpy + * Add broadcast + * Performance improvements: * Improve str.endswith([tuple]) and str.startswith([tuple]) to allow JITting @@ -119,14 +139,18 @@ * Remove the forced minor collection that occurs when rewriting the assembler at the start of the JIT backend + * Port the resource module to cffi + * Internal refactorings: * Use a simpler logger to speed up translation * Drop vestiges of Python 2.5 support in testing + * Update rpython functions with ones needed for py3k + .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.0.0.html -.. _`blog post`: http://morepypy.blogspot.com/2016/02/c-api-support-update.html +.. _Numpy: https://bitbucket.org/pypy/numpy Please update, and continue to help us make PyPy better. diff --git a/pypy/doc/release-5.1.1.rst b/pypy/doc/release-5.1.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-5.1.1.rst @@ -0,0 +1,45 @@ +========== +PyPy 5.1.1 +========== + +We have released a bugfix for PyPy 5.1, due to a regression_ in +installing third-party packages dependant on numpy (using our numpy fork +available at https://bitbucket.org/pypy/numpy ). + +Thanks to those who reported the issue. We also fixed a regression in +translating PyPy which increased the memory required to translate. Improvement +will be noticed by downstream packagers and those who translate rather than +download pre-built binaries. + +.. _regression: https://bitbucket.org/pypy/pypy/issues/2282 + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -0,0 +1,194 @@ +============ +PyPy2.7 v5.3 +============ + +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. +This release includes further improvements for the CAPI compatibility layer +which we call cpyext. In addtion to complete support for lxml, we now pass +most (more than 90%) of the upstream numpy test suite, and much of SciPy is +supported as well. + +We also improved the speed of ... and ... + +We updated cffi_ to ... + +You can download the PyPy2.7 v5.3 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy +.. _cffi: https://cffi.readthedocs.org +.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html +.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.1 released in April 2016) +========================================================= + +* New features: + + * Merge a major expansion of the C-API support in cpyext, here are some of + the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible + - fix many edge cases exposed by fixing tests to run with -A + - issequence() logic matches cpython + - make PyStringObject and PyUnicodeObject field names compatible with cpython + - add prelminary support for PyDateTime_* + - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, + PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, + PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo + - improve support for PyGILState_Ensure, PyGILState_Release, and thread + primitives, also find a case where CPython will allow thread creation + before PyEval_InitThreads is run, dissallow on PyPy + - create a PyObject-specific list strategy + - rewrite slot assignment for typeobjects + - improve tracking of PyObject to rpython object mapping + - support tp_as_{number, sequence, mapping, buffer} slots + - support ByteArrayObject via the new resizable_list_supporting_raw_ptr + - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's + - fix the signature of PyUFunc_FromFuncAndDataAndSignature + - implement many PyWhatever_FOO() as a macro taking a `void *` + + * CPyExt tweak: instead of "GIL not held when a CPython C extension module + calls PyXxx", we now silently acquire/release the GIL. Helps with + CPython C extension modules that call some PyXxx() functions without + holding the GIL (arguably, they are theorically buggy). + + * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. + It is a more flexible way to make RPython finalizers. Use this mechanism to + clean up handling of ``__del__`` methods, fixing issue #2287 + + * Generalize cpyext old-style buffers to more than just str/buffer, add + support for mmap + + * Support command line -v to trace import statements + + * Add rposix functions for PyPy3.3 support + + * Give super an __init__ and a simple __new__ for CPython compatibility + + * Revive traceviewer, a tool to use pygame to view traces + + * Update to cffi/847bbc0297f8 which improves help() on cffi objects + +* Bug Fixes + + * Fix issue #2277: only special-case two exact lists in zip(), not list + subclasses, because an overridden __iter__() should be called (probably) + + * Fix issue #2226: Another tweak in the incremental GC- this should ensure + that progress in the major GC occurs quickly enough in all cases. + + * Clarify and refactor documentation on http://doc.pypy.org + + * Use "must be unicode, not %T" in unicodedata TypeErrors. + + * Manually reset sys.settrace() and sys.setprofile() when we're done running. + This is not exactly what CPython does, but if we get an exception, unlike + CPython, we call functions from the 'traceback' module, and these would + call more the trace/profile function. That's unexpected and can lead + to more crashes at this point. + + * Use the appropriate tp_dealloc on a subclass of a builtin type, and call + tp_new for a python-sublcass of a C-API type + + * Fix for issue #2285 - rare vmprof segfaults on OS/X + + * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc + + * Fix issue #2311 - grab the `__future__` flags imported in the main script, in + `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* Numpy_: + + * Implement ufunc.outer on numpypy + + * Move PyPy-specific numpy headers to a subdirectory (also changed pypy/numpy + accordingly) + +* Performance improvements: + + * Use bitstrings to compress lists of descriptors that are attached to an + EffectInfo + + * Remove most of the _ovf, _zer and _val operations from RPython. Kills + quite some code internally, and allows the JIT to do better + optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` + can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly + negative. + + * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())` + + * Use the __builtin_add_overflow built-ins if they are available + + * Rework the way registers are moved/spilled in before_call() + +* Internal refactorings: + + * Refactor code to better support Python3-compatible syntax + + * Document and refactor OperationError -> oefmt + + * Reduce the size of generated C sources during translation by + eliminating many many unused struct declarations (Issue #2281) + + * Remove a number of translation-time options that were not tested and + never used. Also fix a performance bug in the method cache + + * Reduce the size of generated code by using the same function objects in + all generated subclasses + + * Share cpyext Py* function wrappers according to the signature, shrining the + translated libpypy.so by about + + * Compile c snippets with -Werror, and fix warnings it exposed + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html +.. _Numpy: https://bitbucket.org/pypy/numpy + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst @@ -0,0 +1,69 @@ +=================== +PyPy3 v5.2 alpha 1 +=================== + +We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the +first release of PyPy which targets Python 3.3 (3.3.5) compatibility. + +We would like to thank all of the people who donated_ to the `py3k proposal`_ +for supporting the work that went into this and future releases. + +You can download the PyPy3.3 v5.2 alpha 1 release here: + + http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2 + +Highlights +========== + +* Python 3.3.5 support! + + - Being an early alpha release, there are some `missing features`_ such as a + `PEP 393-like space efficient string representation`_ and `known issues`_ + including performance regressions (e.g. issue `#2305`_). The focus for this + release has been updating to 3.3 compatibility. Windows is also not yet + supported. + +* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4). + +What is PyPy? +============== + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT +compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems except Windows + (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Please try it out and let us know what you think. We welcome feedback, we know +you are using PyPy, please tell us about it! + +We'd especially like to thank these people for their contributions to this +release: + +Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey, +Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others. + +Cheers + +The PyPy Team + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation +.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement +.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29 +.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305 +.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip +.. _`dynamic languages`: http://pypyjs.org diff --git a/pypy/doc/tool/mydot.py b/pypy/doc/tool/mydot.py --- a/pypy/doc/tool/mydot.py +++ b/pypy/doc/tool/mydot.py @@ -68,7 +68,7 @@ help="output format") options, args = parser.parse_args() if len(args) != 1: - raise ValueError, "need exactly one argument" + raise ValueError("need exactly one argument") epsfile = process_dot(py.path.local(args[0])) if options.format == "ps" or options.format == "eps": print epsfile.read() diff --git a/pypy/doc/whatsnew-5.1.0.rst b/pypy/doc/whatsnew-5.1.0.rst --- a/pypy/doc/whatsnew-5.1.0.rst +++ b/pypy/doc/whatsnew-5.1.0.rst @@ -60,3 +60,13 @@ Remove old uneeded numpy headers, what is left is only for testing. Also generate pypy_numpy.h which exposes functions to directly use micronumpy ndarray and ufuncs + +.. branch: rposix-for-3 + +Reuse rposix definition of TIMESPEC in rposix_stat, add wrapper for fstatat(). +This updates the underlying rpython functions with the ones needed for the +py3k branch + +.. branch: numpy_broadcast + +Add broadcast to micronumpy diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,20 +3,143 @@ ========================= .. this is a revision shortly after release-5.1 -.. startrev: 2180e1eaf6f6 +.. startrev: aa60332382a1 -.. branch: rposix-for-3 +.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046 -Reuse rposix definition of TIMESPEC in rposix_stat, add wrapper for fstatat(). -This updates the underlying rpython functions with the ones needed for the -py3k branch - -.. branch: numpy_broadcast +.. branch: gcheader-decl -Add broadcast to micronumpy +Reduce the size of generated C sources. -.. branch: z196-support -Fixes a critical issue in the register allocator and extends support on s390x. PyPy runs and translates on -the s390x revisions z10 (released February 2008, experimental) and z196 (released August 2010) -) in addition to zEC12 and z13. +.. branch: remove-objspace-options + +Remove a number of options from the build process that were never tested and +never set. Fix a performance bug in the method cache. + +.. branch: bitstring + +JIT: use bitstrings to compress the lists of read or written descrs +that we attach to EffectInfo. Fixes a problem we had in +remove-objspace-options. + +.. branch: cpyext-for-merge + +Update cpyext C-API support After this branch, we are almost able to support +upstream numpy via cpyext, so we created (yet another) fork of numpy at +github.com/pypy/numpy with the needed changes. Among the significant changes +to cpyext: + - allow c-snippet tests to be run with -A so we can verify we are compatible + - fix many edge cases exposed by fixing tests to run with -A + - issequence() logic matches cpython + - make PyStringObject and PyUnicodeObject field names compatible with cpython + - add prelminary support for PyDateTime_* + - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, + PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, + - PyAnySet_CheckExact, PyUnicode_Concat + - improve support for PyGILState_Ensure, PyGILState_Release, and thread + primitives, also find a case where CPython will allow thread creation + before PyEval_InitThreads is run, dissallow on PyPy + - create a PyObject-specific list strategy + - rewrite slot assignment for typeobjects + - improve tracking of PyObject to rpython object mapping + - support tp_as_{number, sequence, mapping, buffer} slots + +(makes the pypy-c bigger; this was fixed subsequently by the +share-cpyext-cpython-api branch) + +.. branch: share-mapdict-methods-2 + +Reduce generated code for subclasses by using the same function objects in all +generated subclasses. + +.. branch: share-cpyext-cpython-api + +.. branch: cpyext-auto-gil + +CPyExt tweak: instead of "GIL not held when a CPython C extension module +calls PyXxx", we now silently acquire/release the GIL. Helps with +CPython C extension modules that call some PyXxx() functions without +holding the GIL (arguably, they are theorically buggy). + +.. branch: cpyext-test-A + +Get the cpyext tests to pass with "-A" (i.e. when tested directly with +CPython). + +.. branch: oefmt + +.. branch: cpyext-werror + +Compile c snippets with -Werror in cpyext + +.. branch: gc-del-3 + +Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. +It is a more flexible way to make RPython finalizers. + +.. branch: unpacking-cpython-shortcut + +.. branch: cleanups + +.. branch: cpyext-more-slots + +.. branch: use-gc-del-3 + +Use the new rgc.FinalizerQueue mechanism to clean up the handling of +``__del__`` methods. Fixes notably issue #2287. (All RPython +subclasses of W_Root need to use FinalizerQueue now.) + +.. branch: ufunc-outer + +Implement ufunc.outer on numpypy + +.. branch: verbose-imports + +Support ``pypy -v``: verbose imports. It does not log as much as +cpython, but it should be enough to help when debugging package layout +problems. + +.. branch: cpyext-macros-cast + +Fix some warnings when compiling CPython C extension modules + +.. branch: syntax_fix + +.. branch: remove-raisingops + +Remove most of the _ovf, _zer and _val operations from RPython. Kills +quite some code internally, and allows the JIT to do better +optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` +can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly +negative. + +.. branch: cpyext-old-buffers + +Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap + +.. branch: numpy-includes + +Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy +This allows building upstream numpy and scipy in pypy via cpyext + +.. branch: traceviewer-common-merge-point-formats + +Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. + +.. branch: cpyext-pickle + +Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch +at cpyext import time + +.. branch: nonmovable-list + +Add a way to ask "give me a raw pointer to this list's +items". Only for resizable lists of primitives. Turns the GcArray +nonmovable, possibly making a copy of it first. + +.. branch: cpyext-ext + +Finish the work already partially merged in cpyext-for-merge. Adds support +for ByteArrayObject using the nonmovable-list, which also enables +buffer(bytearray()) diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst @@ -0,0 +1,10 @@ +================================= +What's new in PyPy3 5.1.1 alpha 1 +================================= + +.. A recent revision, ignoring all other branches for this release +.. startrev: 29d14733e007 + +.. branch: py3.3 + +Python 3.3 compatibility diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -238,6 +238,15 @@ for use. The release packaging script will pick up the tcltk runtime in the lib directory and put it in the archive. +The lzma compression library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python 3.3 ship with CFFI wrappers for the lzma library, which can be +downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version +5.0.5, a prebuilt version can be downloaded from +http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature +http://tukaani.org/xz/xz-5.0.5-windows.zip.sig + Using the mingw compiler ------------------------ diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -9,7 +9,7 @@ from rpython.config.config import to_optparse, make_dict, SUPPRESS_USAGE from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace -from pypy.conftest import pypydir +from pypy import pypydir from rpython.rlib import rthread from pypy.module.thread import os_thread @@ -63,7 +63,7 @@ ## from pypy.interpreter import main, interactive, error ## con = interactive.PyPyConsole(space) ## con.interact() - except OperationError, e: + except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) @@ -71,7 +71,7 @@ finally: try: space.finish() - except OperationError, e: + except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) @@ -115,7 +115,7 @@ space.wrap('__import__')) space.call_function(import_, space.wrap('site')) return rffi.cast(rffi.INT, 0) - except OperationError, e: + except OperationError as e: if verbose: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) @@ -167,7 +167,7 @@ sys._pypy_execute_source.append(glob) exec stmt in glob """) - except OperationError, e: + except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) @@ -293,7 +293,7 @@ self.hack_for_cffi_modules(driver) return self.get_entry_point(config) - + def hack_for_cffi_modules(self, driver): # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports @@ -320,7 +320,7 @@ while not basedir.join('include').exists(): _basedir = basedir.dirpath() if _basedir == basedir: From pypy.commits at gmail.com Sun Jun 5 15:57:25 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:25 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: fix merge Message-ID: <57548425.0654c20a.c16a9.ffffcb31@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84946:1123f6abacd1 Date: 2016-06-05 22:50 +0300 http://bitbucket.org/pypy/pypy/changeset/1123f6abacd1/ Log: fix merge diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py --- a/pypy/module/select/test/test_epoll.py +++ b/pypy/module/select/test/test_epoll.py @@ -20,10 +20,6 @@ self.w_sockets = self.space.wrap([]) if platform.machine().startswith('arm'): self.w_timeout = self.space.wrap(0.06) - if platform.machine().startswith('s390x'): - # s390x is not slow, but it seems there is one case when epoll - # modify method is called that takes longer on s390x - self.w_timeout = self.space.wrap(0.06) else: self.w_timeout = self.space.wrap(0.02) From pypy.commits at gmail.com Sun Jun 5 15:57:27 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:27 -0700 (PDT) Subject: [pypy-commit] pypy default: restart whatsnew Message-ID: <57548427.6150c20a.73aed.ffffc5fa@mx.google.com> Author: Matti Picus Branch: Changeset: r84947:8aa9ba306f2e Date: 2016-06-05 22:53 +0300 http://bitbucket.org/pypy/pypy/changeset/8aa9ba306f2e/ Log: restart whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst rename from pypy/doc/whatsnew-head.rst rename to pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -1,5 +1,5 @@ ========================= -What's new in PyPy 5.1+ +What's new in PyPy2.7 5.3 ========================= .. this is a revision shortly after release-5.1 From pypy.commits at gmail.com Sun Jun 5 15:57:21 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:21 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: fix merge Message-ID: <57548421.04251c0a.b4768.ffffec3e@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84944:b7d93985eb7b Date: 2016-06-05 22:47 +0300 http://bitbucket.org/pypy/pypy/changeset/b7d93985eb7b/ Log: fix merge diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.1.2" -#define PYPY_VERSION_NUM 0x05010200 +#define PYPY_VERSION "5.3.0" +#define PYPY_VERSION_NUM 0x05030000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 1, 2, "final", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 0, "final", 0) #XXX # sync patchlevel.h import pypy diff --git a/rpython/doc/arch/index.rst b/rpython/doc/arch/index.rst deleted file mode 100644 --- a/rpython/doc/arch/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _arch_index: - -Architecture specific notes -=========================== - -Here you can find some architecture specific notes. - -.. toctree:: - :maxdepth: 1 - - s390x diff --git a/rpython/doc/arch/s390x.rst b/rpython/doc/arch/s390x.rst deleted file mode 100644 --- a/rpython/doc/arch/s390x.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _s390x: - -IBM Mainframe S390X -=================== - -Our JIT implements the 64 bit version of the IBM Mainframe called s390x. -Note that this architecture is big endian. - -Currently supported ISAs: - -* z13 (released January 2015) -* zEC12 (released September 2012) -* z196 (released August 2010) -* z10 (released February 2008) - -To check if all the necessary CPU facilities are installed -on the subject machine, please run the test using a copy of the pypy -source code:: - - $ ./pytest.py rpython/jit/backend/zarch/test/test_assembler -v -k 'test_facility' - -In addition you can run the auto encoding test to check if your Linux GCC tool chain -is able to compile all instructions used in the JIT backend:: - - $ ./pytest.py rpython/jit/backend/zarch/test/test_auto_encoding.py -v - -Translating ------------ - -Specifically check for these two dependencies. On old versions of some -Linux distributions ship older versions. - -* libffi (version should do > 3.0.+). -* CPython 2.7.+. diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -37,6 +37,7 @@ arm logging + s390x Writing your own interpreter in RPython @@ -60,7 +61,6 @@ getting-started dir-reference jit/index - arch/index translation rtyper garbage_collection diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -44,6 +44,7 @@ # rotating 'RISBG': ('rie_f', ['\xEC','\x55']), + 'RISBGN': ('rie_f', ['\xEC','\x59']), # invert & negative & absolute 'LPGR': ('rre', ['\xB9','\x00']), diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py --- a/rpython/jit/backend/zarch/test/test_assembler.py +++ b/rpython/jit/backend/zarch/test/test_assembler.py @@ -155,15 +155,7 @@ s64 = bin(fac_data[1])[2:] print(f64) print(s64) - for i,c in enumerate(f64): - print('index: %d is set? %s' % (i,c)) - - assert f64[1] == '1' # The z/Architecture architectural mode is installed. - assert f64[2] == '1' # The z/Architecture architectural mode is active. assert f64[18] == '1' # long displacement facility - assert f64[21] == '1' # extended immediate facility - assert f64[34] == '1' # general instruction facility - assert f64[41] == '1' # floating-point-support-enhancement def test_load_byte_zero_extend(self): adr = self.a.datablockwrapper.malloc_aligned(16, 16) @@ -197,7 +189,7 @@ @py.test.mark.parametrize('p', [2**32,2**32+1,2**63-1,2**63-2,0,1,2,3,4,5,6,7,8,10001]) def test_align_withroll(self, p): self.a.mc.load_imm(r.r2, p & 0xffffFFFFffffFFFF) - self.a.mc.RISBG(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) + self.a.mc.RISBGN(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) self.a.mc.BCR(con.ANY, r.r14) assert run_asm(self.a) == rffi.cast(rffi.ULONG,p) & ~(7) @@ -222,7 +214,7 @@ n = 13 l = loc self.a.mc.load_imm(r.r2, 7< Author: Matti Picus Branch: Changeset: r84948:34687aab5146 Date: 2016-06-05 22:54 +0300 http://bitbucket.org/pypy/pypy/changeset/34687aab5146/ Log: finish restart whatsnew-head, increment version diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-head.rst @@ -0,0 +1,7 @@ +========================= +What's new in PyPy2.7 5.3+ +========================= + +.. this is a revision shortly after release-pypy2.7-v5.3 +.. startrev: 873218a739f1 + diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.3.0-alpha0" -#define PYPY_VERSION_NUM 0x05030000 +#define PYPY_VERSION "5.3.1-alpha0" +#define PYPY_VERSION_NUM 0x05030100 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 3, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 1, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Sun Jun 5 15:57:23 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 12:57:23 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: fix merge Message-ID: <57548423.cc1d1c0a.a90ca.ffff982d@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84945:bfd2d6afe938 Date: 2016-06-05 22:48 +0300 http://bitbucket.org/pypy/pypy/changeset/bfd2d6afe938/ Log: fix merge diff --git a/rpython/doc/s390x.rst b/rpython/doc/s390x.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/s390x.rst @@ -0,0 +1,20 @@ +.. _s390x: + +S390X JIT Backend +================= + +Our JIT implements the 64 bit version of the IBM Mainframe called s390x. +Note that this architecture is big endian. + +The following facilities need to be installed to operate +correctly (all of the machines used for development these where installed): + +* General-Instructions-Extension +* Long-Displacement +* Binary Floating Point (IEEE) + +Translating +----------- + +Ensure that libffi is installed (version should do > 3.0.+). +CPython should be version 2.7.+. From pypy.commits at gmail.com Sun Jun 5 15:59:58 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 12:59:58 -0700 (PDT) Subject: [pypy-commit] cffi default: Test and fix: the "bool" type in C++ Message-ID: <575484be.4fa51c0a.d046d.ffff9af2@mx.google.com> Author: Armin Rigo Branch: Changeset: r2707:c417e08b79b7 Date: 2016-06-05 22:01 +0200 http://bitbucket.org/cffi/cffi/changeset/c417e08b79b7/ Log: Test and fix: the "bool" type in C++ diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -57,6 +57,12 @@ # define _CFFI_UNUSED_FN /* nothing */ #endif +#ifdef __cplusplus +# ifndef _Bool +# define _Bool bool /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + /********** CPython-specific section **********/ #ifndef PYPY_VERSION diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -26,6 +26,10 @@ *numbers* instead of *characters*. (Now it is implemented with just a memcpy, of course, not actually iterating over the characters.) +* C++: compiling the generated C code with C++ is supposed to work, + but failed if you make use the ``bool`` type (because that is rendered + as the C ``_Bool`` type, which doesn't exist in C++). + v1.6 ==== diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -1908,3 +1908,10 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + +def test_bool_in_cpp(): + # this works when compiled as C, but in cffi < 1.7 it fails as C++ + ffi = FFI() + ffi.cdef("bool f(void);") + lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }") + assert lib.f() == 1 From pypy.commits at gmail.com Sun Jun 5 16:09:12 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 13:09:12 -0700 (PDT) Subject: [pypy-commit] pypy default: blindly add rel=pypy2-v$maj.$min.$rev to release script Message-ID: <575486e8.a91cc20a.9ae0a.036f@mx.google.com> Author: Matti Picus Branch: Changeset: r84949:ef34203cb42b Date: 2016-06-05 23:08 +0300 http://bitbucket.org/pypy/pypy/changeset/ef34203cb42b/ Log: blindly add rel=pypy2-v$maj.$min.$rev to release script diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,7 +1,7 @@ # Edit these appropriately before running this script maj=5 -min=1 -rev=2 +min=3 +rev=0 branchname=release-$maj.x # ==OR== release-$maj.$min.x tagname=release-$maj.$min.$rev # ==OR== release-$maj.$min @@ -10,6 +10,7 @@ echo checking hg log -r $tagname hg log -r $tagname || exit 1 +rel=pypy2-v$maj.$min.$rev # This script will download latest builds from the buildmaster, rename the top # level directory, and repackage ready to be uploaded to bitbucket. It will also # download source, assuming a tag for the release already exists, and repackage them. @@ -24,27 +25,27 @@ if [ $plat = linux ]; then plat_final=linux32 fi - mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat_final + mv pypy-c-jit-*-$plat $rel-$plat_final echo packaging $plat_final - tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-$plat_final.tar.bz2 pypy-$maj.$min.$rev-$plat_final - rm -rf pypy-$maj.$min.$rev-$plat_final + tar --owner=root --group=root --numeric-owner -cjf $rel-$plat_final.tar.bz2 $rel-$plat_final + rm -rf $rel-$plat_final done plat=win32 wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip unzip pypy-c-jit-latest-$plat.zip -mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat -zip -r pypy-$maj.$min.$rev-$plat.zip pypy-$maj.$min.$rev-$plat -rm -rf pypy-$maj.$min.$rev-$plat +mv pypy-c-jit-*-$plat $rel-$plat +zip -r $rel-$plat.zip $rel-$plat +rm -rf $rel-$plat # Do this after creating a tag, note the untarred directory is pypy-pypy- # so make sure there is not another one wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2 tar -xf $tagname.tar.bz2 -mv pypy-pypy-* pypy-$maj.$min.$rev-src -tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-src.tar.bz2 pypy-$maj.$min.$rev-src -zip -r pypy-$maj.$min.$rev-src.zip pypy-$maj.$min.$rev-src -rm -rf pypy-$maj.$min.$rev-src +mv pypy-pypy-* $rel-src +tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src +zip -r $rel-src.zip $rel-src +rm -rf $rel-src # Print out the md5, sha1, sha256 md5sum *.bz2 *.zip From pypy.commits at gmail.com Sun Jun 5 16:32:52 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 13:32:52 -0700 (PDT) Subject: [pypy-commit] pypy default: update contributor list, two new contributors Message-ID: <57548c74.d02d1c0a.9432d.2351@mx.google.com> Author: Matti Picus Branch: Changeset: r84950:6ccd15eec7cf Date: 2016-06-05 23:32 +0300 http://bitbucket.org/pypy/pypy/changeset/6ccd15eec7cf/ Log: update contributor list, two new contributors diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -43,17 +43,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -93,9 +93,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -104,17 +104,20 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -122,13 +125,13 @@ Simon Cross Edd Barrett Andreas Stührk + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -140,7 +143,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -156,11 +158,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -171,9 +175,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -183,8 +187,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -208,11 +210,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -228,7 +230,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -270,8 +271,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -295,9 +297,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -13,17 +13,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -63,9 +63,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -74,31 +74,34 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross + Edd Barrett Andreas Stührk - Edd Barrett + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -110,7 +113,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -126,11 +128,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -141,9 +145,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -153,8 +157,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -178,11 +180,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -198,7 +200,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -240,8 +241,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -265,9 +267,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -73,6 +73,8 @@ 'Richard Lancaster':['richardlancaster'], 'William Leslie':['William ML Leslie'], 'Spenser Bauman':['Spenser Andrew Bauman'], + 'Raffael Tfirst':['raffael.tfirst at gmail.com'], + 'timo':['timo at eistee.fritz.box'], } alias_map = {} From pypy.commits at gmail.com Sun Jun 5 16:54:03 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 13:54:03 -0700 (PDT) Subject: [pypy-commit] pypy default: Update NumPy project ideas Message-ID: <5754916b.011f1c0a.332fa.ffffa9df@mx.google.com> Author: Matti Picus Branch: Changeset: r84953:2e6fa8685c3d Date: 2016-06-05 23:53 +0300 http://bitbucket.org/pypy/pypy/changeset/2e6fa8685c3d/ Log: Update NumPy project ideas diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -53,15 +53,17 @@ immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated. -Numpy improvements ------------------- +NumPy rebooted +-------------- -The numpy is rapidly progressing in pypy, so feel free to come to IRC and -ask for proposed topic. A not necesarilly up-to-date `list of topics`_ -is also available. +Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. +Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy +test suite. We could use help analyzing the failures and fixing them either +as patches to upstream NumPy, or as fixes to PyPy. -.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt - +We also are looking for help in how to hijack NumPy dtype conversion and +ufunc calls to allow the JIT to make them fast, using our internal _numpypy +module. Improving the jitviewer ------------------------ From pypy.commits at gmail.com Sun Jun 5 16:53:58 2016 From: pypy.commits at gmail.com (pjenvey) Date: Sun, 05 Jun 2016 13:53:58 -0700 (PDT) Subject: [pypy-commit] pypy default: mention last week's PyPy3.3 release, rST fix Message-ID: <57549166.029a1c0a.623ff.ffffa968@mx.google.com> Author: Philip Jenvey Branch: Changeset: r84951:d9d1bac12baa Date: 2016-06-05 13:53 -0700 http://bitbucket.org/pypy/pypy/changeset/d9d1bac12baa/ Log: mention last week's PyPy3.3 release, rST fix diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -2,11 +2,12 @@ PyPy2.7 v5.3 ============ -We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. -This release includes further improvements for the CAPI compatibility layer -which we call cpyext. In addtion to complete support for lxml, we now pass -most (more than 90%) of the upstream numpy test suite, and much of SciPy is -supported as well. +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after +`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3 +compatibility. This new PyPy2.7 release includes further improvements for the +CAPI compatibility layer which we call cpyext. In addtion to complete support +for lxml, we now pass most (more than 90%) of the upstream numpy test suite, +and much of SciPy is supported as well. We also improved the speed of ... and ... @@ -65,6 +66,7 @@ * Merge a major expansion of the C-API support in cpyext, here are some of the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible - fix many edge cases exposed by fixing tests to run with -A - issequence() logic matches cpython @@ -183,6 +185,7 @@ * Compile c snippets with -Werror, and fix warnings it exposed +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html .. _Numpy: https://bitbucket.org/pypy/numpy From pypy.commits at gmail.com Sun Jun 5 16:54:01 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 13:54:01 -0700 (PDT) Subject: [pypy-commit] pypy default: index new files Message-ID: <57549169.4fa81c0a.848ae.fffff5af@mx.google.com> Author: Matti Picus Branch: Changeset: r84952:4618521c911c Date: 2016-06-05 23:52 +0300 http://bitbucket.org/pypy/pypy/changeset/4618521c911c/ Log: index new files diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst release-5.0.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst whatsnew-4.0.1.rst From pypy.commits at gmail.com Sun Jun 5 17:03:37 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 14:03:37 -0700 (PDT) Subject: [pypy-commit] cffi default: Rename 'ffi' to 'ffibuilder' in the docs and in a few demos, when Message-ID: <575493a9.078e1c0a.d6fc3.ffffbfb0@mx.google.com> Author: Armin Rigo Branch: Changeset: r2708:c455de81882f Date: 2016-06-05 23:01 +0200 http://bitbucket.org/cffi/cffi/changeset/c455de81882f/ Log: Rename 'ffi' to 'ffibuilder' in the docs and in a few demos, when it is used in out-of-line builders. I think it makes things clearer, particularly in examples where the two 'ffi' are close together (and even sometimes used in the same sentence...) diff --git a/demo/bsdopendirtype_build.py b/demo/bsdopendirtype_build.py --- a/demo/bsdopendirtype_build.py +++ b/demo/bsdopendirtype_build.py @@ -1,7 +1,7 @@ from cffi import FFI -ffi = FFI() -ffi.cdef(""" +ffibuilder = FFI() +ffibuilder.cdef(""" typedef ... DIR; struct dirent { unsigned char d_type; /* type of file */ @@ -14,10 +14,10 @@ static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; """) -ffi.set_source("_bsdopendirtype", """ +ffibuilder.set_source("_bsdopendirtype", """ #include #include """) if __name__ == '__main__': - ffi.compile() + ffibuilder.compile(verbose=True) diff --git a/demo/bsdopendirtype_setup.py b/demo/bsdopendirtype_setup.py --- a/demo/bsdopendirtype_setup.py +++ b/demo/bsdopendirtype_setup.py @@ -6,7 +6,7 @@ py_modules=["bsdopendirtype"], setup_requires=["cffi>=1.0.dev0"], cffi_modules=[ - "bsdopendirtype_build.py:ffi", + "bsdopendirtype_build.py:ffibuilder", ], install_requires=["cffi>=1.0.dev0"], # should maybe be "cffi-backend" only? zip_safe=False, diff --git a/demo/embedding.py b/demo/embedding.py --- a/demo/embedding.py +++ b/demo/embedding.py @@ -1,12 +1,12 @@ import cffi -ffi = cffi.FFI() +ffibuilder = cffi.FFI() -ffi.embedding_api(""" +ffibuilder.embedding_api(""" int add(int, int); """) -ffi.embedding_init_code(""" +ffibuilder.embedding_init_code(""" from _embedding_cffi import ffi print("preparing") # printed once @@ -16,6 +16,6 @@ return x + y """) -ffi.set_source("_embedding_cffi", "") +ffibuilder.set_source("_embedding_cffi", "") -ffi.compile(verbose=True) +ffibuilder.compile(verbose=True) diff --git a/demo/gmp_build.py b/demo/gmp_build.py --- a/demo/gmp_build.py +++ b/demo/gmp_build.py @@ -6,9 +6,9 @@ # http://bazaar.launchpad.net/~tolot-solar-empire/+junk/gmpy_cffi/files # -ffi = cffi.FFI() +ffibuilder = cffi.FFI() -ffi.cdef(""" +ffibuilder.cdef(""" typedef struct { ...; } MP_INT; typedef MP_INT mpz_t[1]; @@ -19,8 +19,8 @@ """) -ffi.set_source('_gmp_cffi', "#include ", +ffibuilder.set_source('_gmp_cffi', "#include ", libraries=['gmp', 'm']) if __name__ == '__main__': - ffi.compile(verbose=True) + ffibuilder.compile(verbose=True) diff --git a/demo/setup.py b/demo/setup.py deleted file mode 100644 --- a/demo/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# A minimal example of setup.py to install a Python module -# together with the custom C extension module generated by CFFI. -# - -from distutils.core import setup -from distutils.extension import Extension -import bsdopendirtype - -setup(name='bsdopendirtype', - py_modules=['bsdopendirtype'], - ext_modules=[bsdopendirtype.ffi.verifier.get_extension()]) diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -29,12 +29,12 @@ # in a separate file "package/foo_build.py" import cffi - ffi = cffi.FFI() - ffi.set_source("package._foo", None) - ffi.cdef("C-like declarations") + ffibuilder = cffi.FFI() + ffibuilder.set_source("package._foo", None) + ffibuilder.cdef("C-like declarations") if __name__ == "__main__": - ffi.compile() + ffibuilder.compile() Running ``python foo_build.py`` produces a file ``_foo.py``, which can then be imported in the main program: @@ -57,12 +57,12 @@ # in a separate file "package/foo_build.py" import cffi - ffi = cffi.FFI() - ffi.set_source("package._foo", "real C code") # <= - ffi.cdef("C-like declarations with '...'") + ffibuilder = cffi.FFI() + ffibuilder.set_source("package._foo", "real C code") # <= + ffibuilder.cdef("C-like declarations with '...'") if __name__ == "__main__": - ffi.compile(verbose=True) + ffibuilder.compile(verbose=True) Running ``python foo_build.py`` produces a file ``_foo.c`` and invokes the C compiler to turn it into a file ``_foo.so`` (or @@ -91,7 +91,7 @@ setup( ..., - ext_modules=[foo_build.ffi.distutils_extension()], + ext_modules=[foo_build.ffibuilder.distutils_extension()], ) For Setuptools (out-of-line, but works in ABI or API mode; @@ -105,7 +105,7 @@ setup( ..., setup_requires=["cffi>=1.0.0"], - cffi_modules=["package/foo_build.py:ffi"], + cffi_modules=["package/foo_build.py:ffibuilder"], install_requires=["cffi>=1.0.0"], ) @@ -121,9 +121,13 @@ page `Using the ffi/lib objects`_ describes the common functionality. It is what you get in the ``from package._foo import ffi`` lines above. On the other hand, the extended ``FFI`` class is the one you get from -``import cffi; ffi = cffi.FFI()``. It has the same functionality (for -in-line use), but also the extra methods described below (to prepare -the FFI). +``import cffi; ffi_or_ffibuilder = cffi.FFI()``. It has the same +functionality (for in-line use), but also the extra methods described +below (to prepare the FFI). NOTE: We use the name ``ffibuilder`` +instead of ``ffi`` in the out-of-line context, when the code is about +producing a ``_foo.so`` file; this is an attempt to distinguish it +from the different ``ffi`` object that you get by later saying +``from _foo import ffi``. .. _`Using the ffi/lib objects`: using.html @@ -143,15 +147,16 @@ apart from writes to global variables. Also, ``lib.__dict__`` does not work before version 1.2 or if ``lib`` happens to declare a name called ``__dict__`` (use instead ``dir(lib)``). The same is true -for ``lib.__class__`` before version 1.4. +for ``lib.__class__``, ``lib.__all__`` and ``lib.__name__`` added +in successive versions. .. _cdef: -ffi.cdef(): declaring types and functions ------------------------------------------ +ffi/ffibuilder.cdef(): declaring types and functions +---------------------------------------------------- -**ffi.cdef(source)**: parses the given C source. +**ffi/ffibuilder.cdef(source)**: parses the given C source. It registers all the functions, types, constants and global variables in the C source. The types can be used immediately in ``ffi.new()`` and other functions. Before you can access the functions and global @@ -326,18 +331,18 @@ ``ffi.dlopen(ctypes.util.find_library('c'))``. -ffi.set_source(): preparing out-of-line modules ------------------------------------------------ +ffibuilder.set_source(): preparing out-of-line modules +------------------------------------------------------ -**ffi.set_source(module_name, c_header_source, [\*\*keywords...])**: +**ffibuilder.set_source(module_name, c_header_source, [\*\*keywords...])**: prepare the ffi for producing out-of-line an external module called ``module_name``. *New in version 1.0.* -``ffi.set_source()`` by itself does not write any file, but merely +``ffibuilder.set_source()`` by itself does not write any file, but merely records its arguments for later. It can therefore be called before or -after ``ffi.cdef()``. +after ``ffibuilder.cdef()``. -In **ABI mode,** you call ``ffi.set_source(module_name, None)``. The +In **ABI mode,** you call ``ffibuilder.set_source(module_name, None)``. The argument is the name (or dotted name inside a package) of the Python module to generate. In this mode, no C compiler is called. @@ -384,7 +389,7 @@ .. code-block:: python - ffi.set_source("mymodule", ''' + ffibuilder.set_source("mymodule", ''' extern "C" { int somefunc(int somearg) { return real_cpp_func(somearg); } } @@ -504,22 +509,23 @@ different declarations). -ffi.compile() etc.: compiling out-of-line modules -------------------------------------------------- +ffibuilder.compile() etc.: compiling out-of-line modules +-------------------------------------------------------- You can use one of the following functions to actually generate the -.py or .c file prepared with ``ffi.set_source()`` and ``ffi.cdef()``. +.py or .c file prepared with ``ffibuilder.set_source()`` and +``ffibuilder.cdef()``. Note that these function won't overwrite a .py/.c file with exactly the same content, to preserve the mtime. In some cases where you need the mtime to be updated anyway, delete the file before calling the functions. -**ffi.compile(tmpdir='.', verbose=False):** +**ffibuilder.compile(tmpdir='.', verbose=False):** explicitly generate the .py or .c file, and (if .c) compile it. The output file is (or are) put in the directory given by ``tmpdir``. In the examples given here, we use -``if __name__ == "__main__": ffi.compile()`` in the build scripts---if +``if __name__ == "__main__": ffibuilder.compile()`` in the build scripts---if they are directly executed, this makes them rebuild the .py/.c file in the current directory. (Note: if a package is specified in the call to ``set_source()``, then a corresponding subdirectory of the ``tmpdir`` @@ -530,13 +536,13 @@ compiler. (This parameter might be changed to True by default in a future release.) -**ffi.emit_python_code(filename):** generate the given .py file (same -as ``ffi.compile()`` for ABI mode, with an explicitly-named file to +**ffibuilder.emit_python_code(filename):** generate the given .py file (same +as ``ffibuilder.compile()`` for ABI mode, with an explicitly-named file to write). If you choose, you can include this .py file pre-packaged in your own distributions: it is identical for any Python version (2 or 3). -**ffi.emit_c_code(filename):** generate the given .c file (for API +**ffibuilder.emit_c_code(filename):** generate the given .c file (for API mode) without compiling it. Can be used if you have some other method to compile it, e.g. if you want to integrate with some larger build system that will compile this file for you. You can also distribute @@ -545,25 +551,25 @@ if produced on a different OS, with a different version of CPython, or with PyPy; it is done with generating the appropriate ``#ifdef``). -**ffi.distutils_extension(tmpdir='build', verbose=True):** for +**ffibuilder.distutils_extension(tmpdir='build', verbose=True):** for distutils-based ``setup.py`` files. Calling this creates the .c file if needed in the given ``tmpdir``, and returns a ``distutils.core.Extension`` instance. For Setuptools, you use instead the line -``cffi_modules=["path/to/foo_build.py:ffi"]`` in ``setup.py``. This +``cffi_modules=["path/to/foo_build.py:ffibuilder"]`` in ``setup.py``. This line asks Setuptools to import and use a helper provided by CFFI, which in turn executes the file ``path/to/foo_build.py`` (as with -``execfile()``) and looks up its global variable called ``ffi``. You +``execfile()``) and looks up its global variable called ``ffibuilder``. You can also say ``cffi_modules=["path/to/foo_build.py:maker"]``, where ``maker`` names a global function; it is called with no argument and is supposed to return a ``FFI`` object. -ffi.include(): combining multiple CFFI interfaces -------------------------------------------------- +ffi/ffibuilder.include(): combining multiple CFFI interfaces +------------------------------------------------------------ -**ffi.include(other_ffi)**: includes the typedefs, structs, unions, +**ffi/ffibuilder.include(other_ffi)**: includes the typedefs, structs, unions, enums and constants defined in another FFI instance. This is meant for large projects where one CFFI-based interface depends on some types declared in a different CFFI-based interface. @@ -574,9 +580,10 @@ object. (You can write several ``cdef()`` calls over the same ``ffi`` from several Python files, if one file would be too large.) -For out-of-line modules, the ``ffi.include(other_ffi)`` line should -occur in the build script, and the ``other_ffi`` argument should be -another FFI that comes from another build script. When the two build +For out-of-line modules, the ``ffibuilder.include(other_ffibuilder)`` +line should +occur in the build script, and the ``other_ffibuilder`` argument should be +another FFI instance that comes from another build script. When the two build scripts are turned into generated files, say ``_ffi.so`` and ``_other_ffi.so``, then importing ``_ffi.so`` will internally cause ``_other_ffi.so`` to be imported. At that point, the real @@ -722,11 +729,11 @@ check. Be sure to have other means of clearing the ``tmpdir`` whenever you change your sources. -* ``source_extension`` has the same meaning as in ``ffi.set_source()``. +* ``source_extension`` has the same meaning as in ``ffibuilder.set_source()``. * The optional ``flags`` argument (ignored on Windows) defaults to ``ffi.RTLD_NOW``; see ``man dlopen``. (With - ``ffi.set_source()``, you would use ``sys.setdlopenflags()``.) + ``ffibuilder.set_source()``, you would use ``sys.setdlopenflags()``.) * The optional ``relative_to`` argument is useful if you need to list local files passed to the C compiler:: diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -51,16 +51,16 @@ # file plugin_build.py import cffi - ffi = cffi.FFI() + ffibuilder = cffi.FFI() with open('plugin.h') as f: - ffi.embedding_api(f.read()) + ffibuilder.embedding_api(f.read()) - ffi.set_source("my_plugin", ''' + ffibuilder.set_source("my_plugin", ''' #include "plugin.h" ''') - ffi.embedding_init_code(""" + ffibuilder.embedding_init_code(""" from my_plugin import ffi @ffi.def_extern() @@ -69,7 +69,7 @@ return p.x + p.y """) - ffi.compile(target="plugin-1.5.*", verbose=True) + ffibuilder.compile(target="plugin-1.5.*", verbose=True) Running the code above produces a *DLL*, i,e, a dynamically-loadable library. It is a file with the extension ``.dll`` on Windows, @@ -82,7 +82,7 @@ Here are some details about the methods used above: -* **ffi.embedding_api(source):** parses the given C source, which +* **ffibuilder.embedding_api(source):** parses the given C source, which declares functions that you want to be exported by the DLL. It can also declare types, constants and global variables that are part of the C-level API of your DLL. @@ -95,9 +95,9 @@ The global variables, on the other hand, are not automatically produced. You have to write their definition explicitly in - ``ffi.set_source()``, as regular C code (see the point after next). + ``ffibuilder.set_source()``, as regular C code (see the point after next). -* **ffi.embedding_init_code(python_code):** this gives +* **ffibuilder.embedding_init_code(python_code):** this gives initialization-time Python source code. This code is copied ("frozen") inside the DLL. At runtime, the code is executed when the DLL is first initialized, just after Python itself is @@ -105,14 +105,14 @@ extra "built-in" module that can be loaded magically without accessing any files, with a line like "``from my_plugin import ffi, lib``". The name ``my_plugin`` comes from the first argument to - ``ffi.set_source()``. This module represents "the caller's C world" + ``ffibuilder.set_source()``. This module represents "the caller's C world" from the point of view of Python. The initialization-time Python code can import other modules or packages as usual. You may have typical Python issues like needing to set up ``sys.path`` somehow manually first. - For every function declared within ``ffi.embedding_api()``, the + For every function declared within ``ffibuilder.embedding_api()``, the initialization-time Python code or one of the modules it imports should use the decorator ``@ffi.def_extern()`` to attach a corresponding Python function to it. @@ -128,7 +128,7 @@ contains code that calls ``exit()``, for example if importing ``site`` fails. This may be worked around in the future. -* **ffi.set_source(c_module_name, c_code):** set the name of the +* **ffibuilder.set_source(c_module_name, c_code):** set the name of the module from Python's point of view. It also gives more C code which will be included in the generated C code. In trivial examples it can be an empty string. It is where you would ``#include`` some @@ -136,14 +136,14 @@ ``CFFI_DLLEXPORT`` is available to this C code: it expands to the platform-specific way of saying "the following declaration should be exported from the DLL". For example, you would put "``extern int - my_glob;``" in ``ffi.embedding_api()`` and "``CFFI_DLLEXPORT int - my_glob = 42;``" in ``ffi.set_source()``. + my_glob;``" in ``ffibuilder.embedding_api()`` and "``CFFI_DLLEXPORT int + my_glob = 42;``" in ``ffibuilder.set_source()``. - Currently, any *type* declared in ``ffi.embedding_api()`` must also + Currently, any *type* declared in ``ffibuilder.embedding_api()`` must also be present in the ``c_code``. This is automatic if this code contains a line like ``#include "plugin.h"`` in the example above. -* **ffi.compile([target=...] [, verbose=True]):** make the C code and +* **ffibuilder.compile([target=...] [, verbose=True]):** make the C code and compile it. By default, it produces a file called ``c_module_name.dll``, ``c_module_name.dylib`` or ``c_module_name.so``, but the default can be changed with the @@ -155,7 +155,7 @@ "``plugin-1.5.*``". For more complicated cases, you can call instead - ``ffi.emit_c_code("foo.c")`` and compile the resulting ``foo.c`` + ``ffibuilder.emit_c_code("foo.c")`` and compile the resulting ``foo.c`` file using other means. CFFI's compilation logic is based on the standard library ``distutils`` package, which is really developed and tested for the purpose of making CPython extension modules, not @@ -189,29 +189,29 @@ need to have your Python code call C code, read more about `Embedding and Extending`_ below. -* ``ffi.embedding_api(source)``: follows the same syntax as - ``ffi.cdef()``, `documented here.`__ You can use the "``...``" +* ``ffibuilder.embedding_api(source)``: follows the same syntax as + ``ffibuilder.cdef()``, `documented here.`__ You can use the "``...``" syntax as well, although in practice it may be less useful than it is for ``cdef()``. On the other hand, it is expected that often the - C sources that you need to give to ``ffi.embedding_api()`` would be + C sources that you need to give to ``ffibuilder.embedding_api()`` would be exactly the same as the content of some ``.h`` file that you want to give to users of your DLL. That's why the example above does this:: with open('foo.h') as f: - ffi.embedding_api(f.read()) + ffibuilder.embedding_api(f.read()) - Note that a drawback of this approach is that ``ffi.embedding_api()`` + Note that a drawback of this approach is that ``ffibuilder.embedding_api()`` doesn't support ``#ifdef`` directives. You may have to use a more convoluted expression like:: with open('foo.h') as f: lines = [line for line in f if not line.startswith('#')] - ffi.embedding_api(''.join(lines)) + ffibuilder.embedding_api(''.join(lines)) As in the example above, you can also use the same ``foo.h`` from - ``ffi.set_source()``:: + ``ffibuilder.set_source()``:: - ffi.set_source('module_name', '#include "foo.h"') + ffibuilder.set_source('module_name', '#include "foo.h"') .. __: using.html#working @@ -281,7 +281,7 @@ * You can avoid the ``LD_LIBRARY_PATH`` issue if you compile ``libmy_plugin.so`` with the path hard-coded inside in the first place. On Linux, this is done by ``gcc -Wl,-rpath=/some/path``. You - would put this option in ``ffi.set_source("my_plugin", ..., + would put this option in ``ffibuilder.set_source("my_plugin", ..., extra_link_args=['-Wl,-rpath=/some/path/to/libpypy'])``. The path can start with ``$ORIGIN`` to mean "the directory where ``libmy_plugin.so`` is". You can then specify a path relative to that @@ -354,7 +354,7 @@ You might have some issues with the file name: for example, on Windows, Python expects the file to be called ``c_module_name.pyd``, but the CFFI-made DLL is called ``target.dll`` instead. The base name -``target`` is the one specified in ``ffi.compile()``, and on Windows +``target`` is the one specified in ``ffibuilder.compile()``, and on Windows the extension is ``.dll`` instead of ``.pyd``. You have to rename or copy the file, or on POSIX use a symlink. @@ -371,7 +371,8 @@ The embedding mode is not incompatible with the non-embedding mode of CFFI. -You can use *both* ``ffi.embedding_api()`` and ``ffi.cdef()`` in the +You can use *both* ``ffibuilder.embedding_api()`` and +``ffibuilder.cdef()`` in the same build script. You put in the former the declarations you want to be exported by the DLL; you put in the latter only the C functions and types that you want to share between C and Python, but not export from @@ -380,10 +381,10 @@ As an example of that, consider the case where you would like to have a DLL-exported C function written in C directly, maybe to handle some cases before calling Python functions. To do that, you must *not* put -the function's signature in ``ffi.embedding_api()``. (Note that this -requires more hacks if you use ``ffi.embedding_api(f.read())``.) You must -only write the custom function definition in ``ffi.set_source()``, and -prefix it with the macro CFFI_DLLEXPORT: +the function's signature in ``ffibuilder.embedding_api()``. (Note that this +requires more hacks if you use ``ffibuilder.embedding_api(f.read())``.) +You must only write the custom function definition in +``ffibuilder.set_source()``, and prefix it with the macro CFFI_DLLEXPORT: .. code-block:: c @@ -399,11 +400,11 @@ .. code-block:: python - ffi.cdef(""" + ffibuilder.cdef(""" extern "Python" int mycb(int); """) - ffi.set_source("my_plugin", """ + ffibuilder.set_source("my_plugin", """ static int mycb(int); /* the callback: forward declaration, to make it accessible from the C code that follows */ @@ -433,7 +434,7 @@ inserted around it. It only happens when ``myfunc()`` calls ``mycb()``. -As the above explanation hints, this is how ``ffi.embedding_api()`` +As the above explanation hints, this is how ``ffibuilder.embedding_api()`` actually implements function calls that directly invoke Python code; here, we have merely decomposed it explicitly, in order to add some custom C code in the middle. diff --git a/doc/source/overview.rst b/doc/source/overview.rst --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -66,19 +66,20 @@ # file "simple_example_build.py" - # Note: this particular example fails before version 1.0.2 - # because it combines variadic function and ABI level. + # Note: we instantiate the same 'cffi.FFI' class as in the previous + # example, but call the result 'ffibuilder' now instead of 'ffi'; + # this is to avoid confusion with the other 'ffi' object you get below from cffi import FFI - ffi = FFI() - ffi.set_source("_simple_example", None) - ffi.cdef(""" + ffibuilder = FFI() + ffibuilder.set_source("_simple_example", None) + ffibuilder.cdef(""" int printf(const char *format, ...); """) if __name__ == "__main__": - ffi.compile() + ffibuilder.compile(verbose=True) Running it once produces ``_simple_example.py``. Your main program only imports this generated module, not ``simple_example_build.py`` @@ -115,7 +116,7 @@ setup( ... setup_requires=["cffi>=1.0.0"], - cffi_modules=["simple_example_build.py:ffi"], + cffi_modules=["simple_example_build.py:ffibuilder"], install_requires=["cffi>=1.0.0"], ) @@ -131,9 +132,9 @@ # file "example_build.py" from cffi import FFI - ffi = FFI() + ffibuilder = FFI() - ffi.set_source("_example", + ffibuilder.set_source("_example", """ // passed to the real C compiler #include #include @@ -142,7 +143,7 @@ # (more arguments like setup.py's Extension class: # include_dirs=[..], extra_objects=[..], and so on) - ffi.cdef(""" // some declarations from the man page + ffibuilder.cdef(""" // some declarations from the man page struct passwd { char *pw_name; ...; // literally dot-dot-dot @@ -151,7 +152,7 @@ """) if __name__ == "__main__": - ffi.compile() + ffibuilder.compile(verbose=True) You need to run the ``example_build.py`` script once to generate "source code" into the file ``_example.c`` and compile this to a @@ -189,7 +190,7 @@ setup( ... setup_requires=["cffi>=1.0.0"], - cffi_modules=["example_build.py:ffi"], + cffi_modules=["example_build.py:ffibuilder"], install_requires=["cffi>=1.0.0"], ) @@ -252,11 +253,11 @@ # file "example_build.py" from cffi import FFI - ffi = FFI() + ffibuilder = FFI() - ffi.cdef("int foo(int *, int *, int);") + ffibuilder.cdef("int foo(int *, int *, int);") - ffi.set_source("_example", + ffibuilder.set_source("_example", """ static int foo(int *buffer_in, int *buffer_out, int x) { @@ -265,7 +266,7 @@ """) if __name__ == "__main__": - ffi.compile() + ffibuilder.compile(verbose=True) .. code-block:: python @@ -301,15 +302,15 @@ .. code-block:: python import cffi - ffi = cffi.FFI() + ffibuilder = cffi.FFI() - ffi.embedding_api(""" + ffibuilder.embedding_api(""" int do_stuff(int, int); """) - ffi.set_source("my_plugin", "") + ffibuilder.set_source("my_plugin", "") - ffi.embedding_init_code(""" + ffibuilder.embedding_init_code(""" from my_plugin import ffi @ffi.def_extern() @@ -318,7 +319,7 @@ return x + y """) - ffi.compile(target="plugin-1.5.*", verbose=True) + ffibuilder.compile(target="plugin-1.5.*", verbose=True) This simple example creates ``plugin-1.5.dll`` or ``plugin-1.5.so`` as a DLL with a single exported function, ``do_stuff()``. You execute diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -333,7 +333,7 @@ is garbage-collected. The object is modified in-place, and the function returns ``None``. *New in version 1.7: ffi.gc(ptr, None)* -Note that this should be avoided for large memory allocations or +Note that ``ffi.gc()`` should be avoided for large memory allocations or for limited resources. This is particularly true on PyPy: its GC does not know how much memory or how many resources the returned ``ptr`` holds. It will only run its GC when enough memory it knows about has diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -358,7 +358,7 @@ "``...;``" in the ``cdef()``; you need to declare it completely in ``cdef()``. You can work around these limitations by writing a C function with a simpler signature in the C header code passed to -``ffi.set_source()``, and have this C function call the real one. +``ffibuilder.set_source()``, and have this C function call the real one. Aside from these limitations, functions and callbacks can receive and return structs. @@ -450,12 +450,12 @@ In the builder script, declare in the cdef a function prefixed with ``extern "Python"``:: - ffi.cdef(""" + ffibuilder.cdef(""" extern "Python" int my_callback(int, int); void library_function(int(*callback)(int, int)); """) - ffi.set_source("_my_example", """ + ffibuilder.set_source("_my_example", """ #include """) @@ -515,14 +515,14 @@ Then you would write this in the build script:: - ffi.cdef(""" + ffibuilder.cdef(""" typedef ... event_t; typedef void (*event_cb_t)(event_t *evt, void *userdata); void event_cb_register(event_cb_t cb, void *userdata); extern "Python" void my_event_callback(event_t *, void *); """) - ffi.set_source("_demo_cffi", """ + ffibuilder.set_source("_demo_cffi", """ #include """) @@ -589,7 +589,7 @@ :: - ffi.set_source("_demo_cffi", """ + ffibuilder.set_source("_demo_cffi", """ #include static void my_event_callback(widget_t *, event_t *); @@ -601,11 +601,11 @@ directly. Here is an example (inefficient in this case, but might be useful if the logic in ``my_algo()`` is much more complex):: - ffi.cdef(""" + ffibuilder.cdef(""" extern "Python" int f(int); int my_algo(int); """) - ffi.set_source("_example_cffi", """ + ffibuilder.set_source("_example_cffi", """ static int f(int); /* the forward declaration */ static int my_algo(int n) { @@ -646,10 +646,10 @@ these attributes must be written alongside the function *header*, and it is fine if the function *implementation* does not repeat them:: - ffi.cdef(""" + ffibuilder.cdef(""" extern "Python+C" int f(int); /* not static */ """) - ffi.set_source("_example_cffi", """ + ffibuilder.set_source("_example_cffi", """ /* the forward declaration, setting a gcc attribute (this line could also be in some .h file, to be included both here and in the other C files of the project) */ @@ -805,12 +805,12 @@ import cffi - ffi = cffi.FFI() - ffi.cdef(""" + ffibuilder = cffi.FFI() + ffibuilder.cdef(""" int (*python_callback)(int how_many, int *values); void *const c_callback; /* pass this const ptr to C routines */ """) - lib = ffi.set_source("_example", """ + ffibuilder.set_source("_example", """ #include #include static int (*python_callback)(int how_many, int *values); @@ -825,6 +825,7 @@ return python_callback(how_many, values); } """) + ffibuilder.compile(verbose=True) .. code-block:: python @@ -872,7 +873,7 @@ or:: - ffi.cdef(""" + ffibuilder.cdef(""" struct foo_s { int (__stdcall *MyFuncPtr)(int, int); }; From pypy.commits at gmail.com Sun Jun 5 17:11:14 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 14:11:14 -0700 (PDT) Subject: [pypy-commit] cffi default: update version number to 1.7 Message-ID: <57549572.85ba1c0a.98c58.ffffa374@mx.google.com> Author: Armin Rigo Branch: Changeset: r2709:f712d03c57b4 Date: 2016-06-05 23:08 +0200 http://bitbucket.org/cffi/cffi/changeset/f712d03c57b4/ Log: update version number to 1.7 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.6.0" +#define CFFI_VERSION "1.7.0" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.6.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.7.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.6.0" -__version_info__ = (1, 6, 0) +__version__ = "1.7.0" +__version_info__ = (1, 7, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.6.0" + "\ncompiled with cffi version: 1.7.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.6' +version = '1.7' # The full version, including alpha/beta/rc tags. -release = '1.6.0' +release = '1.7.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -51,11 +51,11 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-1.6.0.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-1.7.0.tar.gz - - MD5: 2fae9160991afefb20ff0fbde3b14faf + - MD5: ... - - SHA: 3161ff5d1e791e86f95de258c93371dff9941c4d + - SHA: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -144,7 +144,7 @@ `Mailing list `_ """, - version='1.6.0', + version='1.7.0', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h']} diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py --- a/testing/cffi0/backend_tests.py +++ b/testing/cffi0/backend_tests.py @@ -1358,8 +1358,8 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("enum foo;") from cffi import __version_info__ - if __version_info__ < (1, 7): - py.test.skip("re-enable me in version 1.7") + if __version_info__ < (1, 8): + py.test.skip("re-enable me in version 1.8") e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) assert str(e.value) == ( "'enum foo' has no values explicitly defined: refusing to guess " From pypy.commits at gmail.com Sun Jun 5 17:11:16 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 14:11:16 -0700 (PDT) Subject: [pypy-commit] cffi default: tweaks Message-ID: <57549574.e520c20a.6f540.ffffabb2@mx.google.com> Author: Armin Rigo Branch: Changeset: r2710:c793ba5e283c Date: 2016-06-05 23:12 +0200 http://bitbucket.org/cffi/cffi/changeset/c793ba5e283c/ Log: tweaks diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -6,8 +6,6 @@ v1.7 ==== -(NOT RELEASED YET) - * ``ffi.gc(p, None)`` removes the destructor on an object previously created by another call to ``ffi.gc()`` @@ -26,7 +24,7 @@ *numbers* instead of *characters*. (Now it is implemented with just a memcpy, of course, not actually iterating over the characters.) -* C++: compiling the generated C code with C++ is supposed to work, +* C++: compiling the generated C code with C++ was supposed to work, but failed if you make use the ``bool`` type (because that is rendered as the C ``_Bool`` type, which doesn't exist in C++). From pypy.commits at gmail.com Sun Jun 5 17:19:17 2016 From: pypy.commits at gmail.com (mattip) Date: Sun, 05 Jun 2016 14:19:17 -0700 (PDT) Subject: [pypy-commit] pypy default: merge heads created by pushing at exactly the same time Message-ID: <57549755.421b1c0a.7265b.2491@mx.google.com> Author: Matti Picus Branch: Changeset: r84954:6cce955990e1 Date: 2016-06-06 00:16 +0300 http://bitbucket.org/pypy/pypy/changeset/6cce955990e1/ Log: merge heads created by pushing at exactly the same time diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -2,11 +2,12 @@ PyPy2.7 v5.3 ============ -We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. -This release includes further improvements for the CAPI compatibility layer -which we call cpyext. In addtion to complete support for lxml, we now pass -most (more than 90%) of the upstream numpy test suite, and much of SciPy is -supported as well. +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after +`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3 +compatibility. This new PyPy2.7 release includes further improvements for the +CAPI compatibility layer which we call cpyext. In addtion to complete support +for lxml, we now pass most (more than 90%) of the upstream numpy test suite, +and much of SciPy is supported as well. We also improved the speed of ... and ... @@ -65,6 +66,7 @@ * Merge a major expansion of the C-API support in cpyext, here are some of the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible - fix many edge cases exposed by fixing tests to run with -A - issequence() logic matches cpython @@ -183,6 +185,7 @@ * Compile c snippets with -Werror, and fix warnings it exposed +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html .. _Numpy: https://bitbucket.org/pypy/numpy From pypy.commits at gmail.com Sun Jun 5 17:27:15 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 14:27:15 -0700 (PDT) Subject: [pypy-commit] cffi default: Python3 compat Message-ID: <57549933.d4d71c0a.5410a.ffffa7ee@mx.google.com> Author: Armin Rigo Branch: Changeset: r2711:490ae5a1fe32 Date: 2016-06-05 23:25 +0200 http://bitbucket.org/cffi/cffi/changeset/490ae5a1fe32/ Log: Python3 compat diff --git a/c/lib_obj.c b/c/lib_obj.c --- a/c/lib_obj.c +++ b/c/lib_obj.c @@ -119,7 +119,7 @@ by calling _cffi_type(). */ PyObject *result = NULL; - CTypeDescrObject **pfargs; + CTypeDescrObject **pfargs = NULL; CTypeDescrObject *fresult; Py_ssize_t nargs = 0; struct CPyExtFunc_s *xfunc; diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py --- a/cffi/backend_ctypes.py +++ b/cffi/backend_ctypes.py @@ -205,9 +205,7 @@ def __nonzero__(self): return bool(self._address) - - def __bool__(self): - return bool(self._address) + __bool__ = __nonzero__ @classmethod def _to_ctypes(cls, value): @@ -465,6 +463,7 @@ else: def __nonzero__(self): return self._value != 0 + __bool__ = __nonzero__ if kind == 'float': @staticmethod From pypy.commits at gmail.com Sun Jun 5 17:37:47 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 14:37:47 -0700 (PDT) Subject: [pypy-commit] pypy default: update to cffi/490ae5a1fe32 Message-ID: <57549bab.a60ac20a.20ce9.ffffe698@mx.google.com> Author: Armin Rigo Branch: Changeset: r84955:6c1fbb439469 Date: 2016-06-05 23:36 +0200 http://bitbucket.org/pypy/pypy/changeset/6c1fbb439469/ Log: update to cffi/490ae5a1fe32 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.6.0 +Version: 1.7.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.6.0" -__version_info__ = (1, 6, 0) +__version__ = "1.7.0" +__version_info__ = (1, 7, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -57,6 +57,12 @@ # define _CFFI_UNUSED_FN /* nothing */ #endif +#ifdef __cplusplus +# ifndef _Bool +# define _Bool bool /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + /********** CPython-specific section **********/ #ifndef PYPY_VERSION diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.6.0" + "\ncompiled with cffi version: 1.7.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -205,9 +205,7 @@ def __nonzero__(self): return bool(self._address) - - def __bool__(self): - return bool(self._address) + __bool__ = __nonzero__ @classmethod def _to_ctypes(cls, value): @@ -465,6 +463,7 @@ else: def __nonzero__(self): return self._value != 0 + __bool__ = __nonzero__ if kind == 'float': @staticmethod diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.6.0" +VERSION = "1.7.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.6.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.7.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1359,8 +1359,8 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("enum foo;") from cffi import __version_info__ - if __version_info__ < (1, 7): - py.test.skip("re-enable me in version 1.7") + if __version_info__ < (1, 8): + py.test.skip("re-enable me in version 1.8") e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) assert str(e.value) == ( "'enum foo' has no values explicitly defined: refusing to guess " diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -1909,3 +1909,10 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + +def test_bool_in_cpp(): + # this works when compiled as C, but in cffi < 1.7 it fails as C++ + ffi = FFI() + ffi.cdef("bool f(void);") + lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }") + assert lib.f() == 1 From pypy.commits at gmail.com Sun Jun 5 17:49:55 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 05 Jun 2016 14:49:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Update BUILD_ methods Message-ID: <57549e83.4e0b1c0a.8ba14.ffffbce2@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84956:f02e8c9c821f Date: 2016-06-05 23:49 +0200 http://bitbucket.org/pypy/pypy/changeset/f02e8c9c821f/ Log: Update BUILD_ methods diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1335,19 +1335,39 @@ def BUILD_TUPLE_UNPACK(self, itemcount, next_instr): w_sum = self.space.newset() - self.pushvalue(w_set) + for i in range(itemcount, 0, -1): + self.space.call_method(w_sum, 'update', self.space.peek(i)) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_sum) def BUILD_LIST_UNPACK(self, itemcount, next_instr): w_sum = self.space.newset() - self.pushvalue(w_set) + for i in range(itemcount, 0, -1): + self.space.call_method(w_sum, 'update', self.space.peek(i)) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_sum) def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): w_sum = self.space.newset() - self.pushvalue(w_set) + for i in range(itemcount, 0, -1): + self.space.call_method(w_sum, 'update', self.space.peek(i)) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_sum) def BUILD_MAP_UNPACK(self, itemcount, next_instr): w_sum = self.space.newset() - self.pushvalue(w_set) + for i in range(itemcount, 0, -1): + self.space.call_method(w_sum, 'update', self.space.peek(i)) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_sum) ### ____________________________________________________________ ### class ExitFrame(Exception): From pypy.commits at gmail.com Sun Jun 5 18:14:47 2016 From: pypy.commits at gmail.com (mjacob) Date: Sun, 05 Jun 2016 15:14:47 -0700 (PDT) Subject: [pypy-commit] pypy default: (mjacob, arigo) Remove dead code chain: RPyString_FromString, RPyString_New, predeclare_utility_functions. Message-ID: <5754a457.c68e1c0a.b5ee1.ffffbdce@mx.google.com> Author: Manuel Jacob Branch: Changeset: r84957:b8761aab407c Date: 2016-06-06 00:09 +0200 http://bitbucket.org/pypy/pypy/changeset/b8761aab407c/ Log: (mjacob, arigo) Remove dead code chain: RPyString_FromString, RPyString_New, predeclare_utility_functions. Fix in rtyper.py by Armin. diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -204,6 +204,9 @@ blockcount = 0 self.annmixlevel = None while True: + # make sure all reprs so far have had their setup() called + self.call_all_setups() + # look for blocks not specialized yet pending = [block for block in self.annotator.annotated if block not in self.already_seen] @@ -236,8 +239,6 @@ previous_percentage = percentage self.log.event('specializing: %d / %d blocks (%d%%)' % (n, total, percentage)) - # make sure all reprs so far have had their setup() called - self.call_all_setups() self.log.event('-=- specialized %d%s blocks -=-' % ( blockcount, newtext)) diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -48,7 +48,6 @@ self.delayedfunctionptrs = [] self.completedcontainers = 0 self.containerstats = {} - self.helpers = OrderedDict() # late_initializations is for when the value you want to # assign to a constant object is something C doesn't think is diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -7,27 +7,6 @@ from rpython.translator.c.support import cdecl -def predeclare_common_types(db, rtyper): - # Common types - yield ('RPyString', STR) - -def predeclare_utility_functions(db, rtyper): - # Common utility functions - def RPyString_New(length=lltype.Signed): - return mallocstr(length) - - for fname, f in locals().items(): - if isinstance(f, types.FunctionType): - # XXX this is painful :( - if fname in db.helpers: - yield (fname, db.helpers[fname]) - else: - # hack: the defaults give the type of the arguments - graph = rtyper.annotate_helper(f, f.func_defaults) - db.helpers[fname] = graph - yield (fname, graph) - - def predeclare_exception_data(exctransformer, rtyper): # Exception-related types and constants exceptiondata = rtyper.exceptiondata @@ -56,11 +35,8 @@ def predeclare_all(db, rtyper): - for fn in [predeclare_common_types, - predeclare_utility_functions, - ]: - for t in fn(db, rtyper): - yield t + # Common types + yield ('RPyString', STR) exctransformer = db.exctransformer for t in predeclare_exception_data(exctransformer, rtyper): diff --git a/rpython/translator/c/src/rtyper.c b/rpython/translator/c/src/rtyper.c --- a/rpython/translator/c/src/rtyper.c +++ b/rpython/translator/c/src/rtyper.c @@ -36,11 +36,3 @@ free(dump); } } - -RPyString *RPyString_FromString(char *buf) -{ - int length = strlen(buf); - RPyString *rps = RPyString_New(length); - memcpy(rps->rs_chars.items, buf, length); - return rps; -} diff --git a/rpython/translator/c/src/rtyper.h b/rpython/translator/c/src/rtyper.h --- a/rpython/translator/c/src/rtyper.h +++ b/rpython/translator/c/src/rtyper.h @@ -11,4 +11,3 @@ RPY_EXTERN char *RPyString_AsCharP(RPyString *rps); RPY_EXTERN void RPyString_FreeCache(void); -RPY_EXTERN RPyString *RPyString_FromString(char *buf); From pypy.commits at gmail.com Sun Jun 5 18:18:41 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 15:18:41 -0700 (PDT) Subject: [pypy-commit] pypy default: Don't look inside, JIT Message-ID: <5754a541.cc1d1c0a.a90ca.ffffc6fc@mx.google.com> Author: Armin Rigo Branch: Changeset: r84958:96dbb1b30bd3 Date: 2016-06-06 00:19 +0200 http://bitbucket.org/pypy/pypy/changeset/96dbb1b30bd3/ Log: Don't look inside, JIT diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1241,6 +1241,7 @@ return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, v_list) + at jit.dont_look_inside def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): """ WARNING: dragons ahead. From pypy.commits at gmail.com Mon Jun 6 02:54:18 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 23:54:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Renames Message-ID: <57551e1a.d4d71c0a.5410a.3239@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84959:76bf391f9e87 Date: 2016-06-06 08:32 +0200 http://bitbucket.org/pypy/pypy/changeset/76bf391f9e87/ Log: Renames diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -278,9 +278,9 @@ BoolOption("split_gc_address_space", "Ensure full separation of GC and non-GC pointers", default=False), - BoolOption("reversedb", + BoolOption("reverse_debugger", "Give an executable that writes a log file for reverse debugging", - default=False, cmdline='--reversedb', + default=False, cmdline='--revdb', requires=[('translation.split_gc_address_space', True), ('translation.jit', False)]), ]) diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -33,12 +33,12 @@ thread_enabled=False, sandbox=False, split_gc_address_space=False, - reversedb=False): + reverse_debugger=False): self.translator = translator self.standalone = standalone self.sandbox = sandbox self.split_gc_address_space = split_gc_address_space - self.reversedb = reversedb + self.reverse_debugger = reverse_debugger if gcpolicyclass is None: gcpolicyclass = gc.RefcountingGcPolicy self.gcpolicy = gcpolicyclass(self, thread_enabled) diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -2,7 +2,6 @@ from rpython.translator.c.support import cdecl from rpython.translator.c.support import llvalue_from_constant, gen_assignments from rpython.translator.c.support import c_string_constant, barebonearray -from rpython.translator.c.primitive import PRIMITIVE_FLOATS, PRIMITIVE_TWO_LONGS from rpython.flowspace.model import Variable, Constant from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned, SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType, @@ -425,16 +424,6 @@ def OP_JIT_CONDITIONAL_CALL(self, op): return 'abort(); /* jit_conditional_call */' - def _reverse_db_emit(self, T, value): - if T is Void: - return '/* rpy_reverse_db_emit_void(%s); */' % (value,) - elif T in PRIMITIVE_FLOATS: - return 'rpy_reverse_db_emit_float(%s);' % (value,) - elif T in PRIMITIVE_TWO_LONGS: - return 'rpy_reverse_db_emit_two_longs(%s);' % (value,) - else: - return 'rpy_reverse_db_emit((Signed)%s);' % (value,) - # low-level operations def generic_get(self, op, sourceexpr): T = self.lltypemap(op.result) @@ -442,10 +431,12 @@ result = '%s = %s;' % (newvalue, sourceexpr) if T is Void: result = '/* %s */' % result - if self.db.reversedb: + if self.db.reverse_debugger: S = self.lltypemap(op.args[0]).TO if S._gckind != 'gc' and not S._hints.get('is_excdata'): - result += '\t' + self._reverse_db_emit(T, newvalue) + from rpython.translator.revdb import revdb_genc + result += '\t' + revdb_genc.emit(self.lltypename(op.result), + newvalue) return result def generic_set(self, op, targetexpr): diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -115,8 +115,8 @@ def get_eci(self): pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] - if self.config.translation.reversedb: - include_dirs.append(pypy_include_dir.join('..', 'reversedb')) + if self.config.translation.reverse_debugger: + include_dirs.append(pypy_include_dir.join('..', 'revdb')) return ExternalCompilationInfo(include_dirs=include_dirs) def build_database(self): @@ -136,7 +136,8 @@ sandbox=self.config.translation.sandbox, split_gc_address_space= self.config.translation.split_gc_address_space, - reversedb=self.config.translation.reversedb) + reverse_debugger= + self.config.translation.reverse_debugger) self.db = db # give the gc a chance to register interest in the start-up functions it @@ -218,8 +219,8 @@ defines['COUNT_OP_MALLOCS'] = 1 if self.config.translation.sandbox: defines['RPY_SANDBOXED'] = 1 - if self.config.translation.reversedb: - defines['RPY_REVERSE_DB'] = 1 + if self.config.translation.reverse_debugger: + defines['RPY_REVERSE_DEBUGGER'] = 1 if CBuilder.have___thread is None: CBuilder.have___thread = self.translator.platform.check___thread() if not self.standalone: @@ -846,9 +847,9 @@ ] if _CYGWIN: files.append(srcdir / 'cygwin_wait.c') - if database.reversedb: - from rpython.translator.reversedb import rdb_genc - files += rdb_genc.extra_files() + if database.reverse_debugger: + from rpython.translator.revdb import revdb_genc + files += revdb_genc.extra_files() return eci.merge(ExternalCompilationInfo(separate_module_files=files)) diff --git a/rpython/translator/c/primitive.py b/rpython/translator/c/primitive.py --- a/rpython/translator/c/primitive.py +++ b/rpython/translator/c/primitive.py @@ -223,11 +223,6 @@ Address: name_address, GCREF: name_gcref, } -PRIMITIVE_FLOATS = set([Float, SingleFloat, LongFloat]) -if SignedLongLong is Signed: - PRIMITIVE_TWO_LONGS = set([]) -else: - PRIMITIVE_TWO_LONGS = set([SignedLongLong, UnsignedLongLong]) PrimitiveType = { SignedLongLong: 'long long @', diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -82,7 +82,7 @@ pypy_asm_stack_bottom(); instrument_setup(); -#ifdef RPY_REVERSE_DB +#ifdef RPY_REVERSE_DEBUGGER rpy_reverse_db_setup(argc, argv); #endif diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h --- a/rpython/translator/c/src/g_include.h +++ b/rpython/translator/c/src/g_include.h @@ -51,6 +51,6 @@ #include "src/cygwin_wait.h" #endif -#ifdef RPY_REVERSE_DB -#include "rdb-src/rdb_include.h" +#ifdef RPY_REVERSE_DEBUGGER +#include "rdb-src/revdb_include.h" #endif diff --git a/rpython/translator/reversedb/__init__.py b/rpython/translator/revdb/__init__.py rename from rpython/translator/reversedb/__init__.py rename to rpython/translator/revdb/__init__.py diff --git a/rpython/translator/reversedb/rdb-src/rdb.c b/rpython/translator/revdb/rdb-src/revdb.c rename from rpython/translator/reversedb/rdb-src/rdb.c rename to rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/reversedb/rdb-src/rdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -4,14 +4,14 @@ #include #include -#include "rdb-src/rdb_include.h" +#include "rdb-src/revdb_include.h" #define RDB_SIGNATURE 0x0A424452 /* "RDB\n" */ #define RDB_VERSION 0x00FF0001 -Signed *rpy_rev_buf_p, *rpy_rev_buf_end; -static Signed rpy_rev_buffer[2048]; +rpy_revdb_t rpy_revdb; +static char rpy_rev_buffer[16384]; static int rpy_rev_fileno = -1; @@ -20,30 +20,29 @@ { /* init-time setup */ - char *filename = getenv("PYPYRDB"); + char *filename = getenv("PYPYREVDB"); - rpy_rev_buf_p = rpy_rev_buffer; - rpy_rev_buf_end = rpy_rev_buffer + - sizeof(rpy_rev_buffer) / sizeof(rpy_rev_buffer[0]); + rpy_revdb.buf_p = rpy_rev_buffer; + rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; if (filename && *filename) { - putenv("PYPYRDB="); + putenv("PYPYREVDB="); rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT | O_NOCTTY | O_TRUNC, 0600); if (rpy_rev_fileno < 0) { - fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n", + fprintf(stderr, "Fatal error: can't create PYPYREVDB file '%s'\n", filename); abort(); } atexit(rpy_reverse_db_flush); } - rpy_reverse_db_emit(RDB_SIGNATURE); - rpy_reverse_db_emit(RDB_VERSION); - rpy_reverse_db_emit(0); - rpy_reverse_db_emit(0); - rpy_reverse_db_emit(argc); - rpy_reverse_db_emit((Signed)argv); + rpy_reverse_db_EMIT(Signed _e=RDB_SIGNATURE); + rpy_reverse_db_EMIT(Signed _e=RDB_VERSION); + rpy_reverse_db_EMIT(Signed _e=0); + rpy_reverse_db_EMIT(Signed _e=0); + rpy_reverse_db_EMIT(Signed _e=argc); + rpy_reverse_db_EMIT(void *_e=argv); } RPY_EXTERN @@ -51,11 +50,11 @@ { /* write the current buffer content to the OS */ - ssize_t size = (rpy_rev_buf_p - rpy_rev_buffer) * sizeof(rpy_rev_buffer[0]); - rpy_rev_buf_p = rpy_rev_buffer; + ssize_t size = rpy_revdb.buf_p - rpy_rev_buffer; + rpy_revdb.buf_p = rpy_rev_buffer; if (size > 0 && rpy_rev_fileno >= 0) { if (write(rpy_rev_fileno, rpy_rev_buffer, size) != size) { - fprintf(stderr, "Fatal error: writing to PYPYRDB file: %m\n"); + fprintf(stderr, "Fatal error: writing to PYPYREVDB file: %m\n"); abort(); } } diff --git a/rpython/translator/reversedb/rdb-src/rdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h rename from rpython/translator/reversedb/rdb-src/rdb_include.h rename to rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/reversedb/rdb-src/rdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -4,29 +4,12 @@ RPY_EXTERN void rpy_reverse_db_flush(void); -RPY_EXTERN Signed *rpy_rev_buf_p, *rpy_rev_buf_end; +typedef struct { char *buf_p, *buf_limit; } rpy_revdb_t; +RPY_EXTERN rpy_revdb_t rpy_revdb; - -static inline void rpy_reverse_db_emit(Signed value) { - *(rpy_rev_buf_p++) = value; - if (rpy_rev_buf_p == rpy_rev_buf_end) - rpy_reverse_db_flush(); -} - -static inline void rpy_reverse_db_emit_float(double value) { - /* xxx for 'long double' this can loose some precision */ - Signed sval[sizeof(double) / SIZEOF_LONG]; - memcpy(sval, &value, sizeof(value)); - rpy_reverse_db_emit(sval[0]); - if (SIZEOF_LONG < sizeof(double)) /* assume len(sval) is exactly 1 or 2 */ - rpy_reverse_db_emit(sval[1]); -} - -static inline void rpy_reverse_db_emit_two_longs(long long value) -{ - Signed sval[2]; - assert(SIZEOF_LONG * 2 == SIZEOF_LONG_LONG); - memcpy(sval, &value, sizeof(value)); - rpy_reverse_db_emit(sval[0]); - rpy_reverse_db_emit(sval[1]); -} +#define rpy_reverse_db_EMIT(decl_e) do { \ + decl_e; \ + memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ + if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ + rpy_reverse_db_flush(); \ +} while (0) diff --git a/rpython/translator/reversedb/rdb_genc.py b/rpython/translator/revdb/revdb_genc.py rename from rpython/translator/reversedb/rdb_genc.py rename to rpython/translator/revdb/revdb_genc.py --- a/rpython/translator/reversedb/rdb_genc.py +++ b/rpython/translator/revdb/revdb_genc.py @@ -1,7 +1,15 @@ import py +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.c.support import cdecl + def extra_files(): srcdir = py.path.local(__file__).join('..', 'rdb-src') return [ - srcdir / 'rdb.c', + srcdir / 'revdb.c', ] + +def emit(tp, value): + if tp == 'void @': + return '/* void */' + return 'rpy_reverse_db_EMIT(%s=%s);' % (cdecl(tp, '_e'), value) diff --git a/rpython/translator/reversedb/test/__init__.py b/rpython/translator/revdb/test/__init__.py rename from rpython/translator/reversedb/test/__init__.py rename to rpython/translator/revdb/test/__init__.py diff --git a/rpython/translator/reversedb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py rename from rpython/translator/reversedb/test/test_basic.py rename to rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/reversedb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -3,37 +3,36 @@ import array, struct from rpython.tool.udir import udir from rpython.translator.interactive import Translation +from rpython.rlib.rarithmetic import LONG_BIT class RDB(object): def __init__(self, filename): - f = open(filename, 'rb') - f.seek(0, 2) - filesize = f.tell() - f.seek(0, 0) - self.items = array.array("l") - self.items.fromfile(f, filesize / struct.calcsize("l")) - f.close() + with open(filename, 'rb') as f: + self.buffer = f.read() # - assert self.items[0] == 0x0A424452 - assert self.items[1] == 0x00FF0001 - assert self.items[2] == 0 - assert self.items[3] == 0 - self.argc = self.items[4] - self.argv = self.items[5] - self.cur = 6 + self.cur = 0 + x = self.next(); assert x == 0x0A424452 + x = self.next(); assert x == 0x00FF0001 + x = self.next(); assert x == 0 + x = self.next(); assert x == 0 + self.argc = self.next() + self.argv = self.next() - def next(self): - n = self.cur - self.cur = n + 1 - return self.items[n] + def next(self, mode='P'): + p = self.cur + self.cur = p + struct.calcsize(mode) + return struct.unpack_from(mode, self.buffer, p)[0] + + def done(self): + return self.cur == len(self.buffer) class TestBasic(object): def getcompiled(self, entry_point, argtypes, backendopt=True): t = Translation(entry_point, None, gc="boehm") - t.config.translation.reversedb = True + t.config.translation.reverse_debugger = True t.config.translation.rweakref = False if not backendopt: t.disable(["backendopt_lltype"]) @@ -47,7 +46,7 @@ def run(*argv): env = os.environ.copy() - env['PYPYRDB'] = self.rdbname + env['PYPYREVDB'] = self.rdbname stdout = t.driver.cbuilder.cmdexec(' '.join(argv), env=env) return stdout return run @@ -70,17 +69,17 @@ s = [] # first we determine the length of the "char *p" while True: - c = rdb.next() - if c == 0: + c = rdb.next('c') + if c == '\x00': break - s.append(chr(c)) + s.append(c) # then we really read the "char *" and copy it into a rpy string # (that's why this time we don't read the final \0) for c1 in s: - c2 = rdb.next() - assert c2 == ord(c1) + c2 = rdb.next('c') + assert c2 == c1 got.append(''.join(s)) # that's all that should get from this simple example - assert rdb.cur == len(rdb.items) + assert rdb.done() # assert got == [self.exename, 'abc', 'd'] From pypy.commits at gmail.com Mon Jun 6 02:54:20 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 23:54:20 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: External function calls Message-ID: <57551e1c.c3381c0a.c9767.36e8@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84960:f7fe5ea0f6d4 Date: 2016-06-06 08:55 +0200 http://bitbucket.org/pypy/pypy/changeset/f7fe5ea0f6d4/ Log: External function calls diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -391,11 +391,16 @@ # skip assignment of 'void' return value r = self.expr(v_result) line = '%s = %s' % (r, line) - if targets: + else: + r = None + if targets is not None: for graph in targets: if getattr(graph, 'inhibit_tail_call', False): line += '\nPYPY_INHIBIT_TAIL_CALL();' break + elif self.db.reverse_debugger: + from rpython.translator.revdb import revdb_genc + line += '\n' + revdb_genc.emit(self.lltypename(v_result), r) return line def OP_DIRECT_CALL(self, op): diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -79,6 +79,9 @@ c2 = rdb.next('c') assert c2 == c1 got.append(''.join(s)) + # write() call + x = rdb.next(); assert x == len('[abc, d]\n') + x = rdb.next('i'); assert x == 0 # errno # that's all that should get from this simple example assert rdb.done() # From pypy.commits at gmail.com Mon Jun 6 02:57:50 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Jun 2016 23:57:50 -0700 (PDT) Subject: [pypy-commit] pypy default: oops Message-ID: <57551eee.86981c0a.7317.3494@mx.google.com> Author: Armin Rigo Branch: Changeset: r84961:251a2846d850 Date: 2016-06-06 08:58 +0200 http://bitbucket.org/pypy/pypy/changeset/251a2846d850/ Log: oops diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -398,8 +398,8 @@ self.rm.force_spill_var(var) def before_call(self, save_all_regs=False): - self.rm.before_call(save_all_regs) - self.vfprm.before_call(save_all_regs) + self.rm.before_call(save_all_regs=save_all_regs) + self.vfprm.before_call(save_all_regs=save_all_regs) def _sync_var(self, v): if v.type == FLOAT: From pypy.commits at gmail.com Mon Jun 6 03:38:05 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 00:38:05 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: merge default into release Message-ID: <5755285d.c39dc20a.2e75e.ffffefc2@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84962:581a2c8a00c5 Date: 2016-06-06 10:35 +0300 http://bitbucket.org/pypy/pypy/changeset/581a2c8a00c5/ Log: merge default into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -43,17 +43,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -93,9 +93,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -104,17 +104,20 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -122,13 +125,13 @@ Simon Cross Edd Barrett Andreas Stührk + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -140,7 +143,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -156,11 +158,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -171,9 +175,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -183,8 +187,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -208,11 +210,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -228,7 +230,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -270,8 +271,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -295,9 +297,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.6.0 +Version: 1.7.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.6.0" -__version_info__ = (1, 6, 0) +__version__ = "1.7.0" +__version_info__ = (1, 7, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -57,6 +57,12 @@ # define _CFFI_UNUSED_FN /* nothing */ #endif +#ifdef __cplusplus +# ifndef _Bool +# define _Bool bool /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + /********** CPython-specific section **********/ #ifndef PYPY_VERSION diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.6.0" + "\ncompiled with cffi version: 1.7.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -205,9 +205,7 @@ def __nonzero__(self): return bool(self._address) - - def __bool__(self): - return bool(self._address) + __bool__ = __nonzero__ @classmethod def _to_ctypes(cls, value): @@ -465,6 +463,7 @@ else: def __nonzero__(self): return self._value != 0 + __bool__ = __nonzero__ if kind == 'float': @staticmethod diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -13,17 +13,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -63,9 +63,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -74,31 +74,34 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross + Edd Barrett Andreas Stührk - Edd Barrett + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -110,7 +113,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -126,11 +128,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -141,9 +145,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -153,8 +157,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -178,11 +180,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -198,7 +200,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -240,8 +241,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -265,9 +267,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst release-5.0.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst whatsnew-4.0.1.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -53,15 +53,17 @@ immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated. -Numpy improvements ------------------- +NumPy rebooted +-------------- -The numpy is rapidly progressing in pypy, so feel free to come to IRC and -ask for proposed topic. A not necesarilly up-to-date `list of topics`_ -is also available. +Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. +Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy +test suite. We could use help analyzing the failures and fixing them either +as patches to upstream NumPy, or as fixes to PyPy. -.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt - +We also are looking for help in how to hijack NumPy dtype conversion and +ufunc calls to allow the JIT to make them fast, using our internal _numpypy +module. Improving the jitviewer ------------------------ diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -2,11 +2,12 @@ PyPy2.7 v5.3 ============ -We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. -This release includes further improvements for the CAPI compatibility layer -which we call cpyext. In addtion to complete support for lxml, we now pass -most (more than 90%) of the upstream numpy test suite, and much of SciPy is -supported as well. +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after +`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3 +compatibility. This new PyPy2.7 release includes further improvements for the +CAPI compatibility layer which we call cpyext. In addtion to complete support +for lxml, we now pass most (more than 90%) of the upstream numpy test suite, +and much of SciPy is supported as well. We also improved the speed of ... and ... @@ -65,6 +66,7 @@ * Merge a major expansion of the C-API support in cpyext, here are some of the highlights: + - allow c-snippet tests to be run with -A so we can verify we are compatible - fix many edge cases exposed by fixing tests to run with -A - issequence() logic matches cpython @@ -183,6 +185,7 @@ * Compile c snippets with -Werror, and fix warnings it exposed +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html .. _Numpy: https://bitbucket.org/pypy/numpy diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -73,6 +73,8 @@ 'Richard Lancaster':['richardlancaster'], 'William Leslie':['William ML Leslie'], 'Spenser Bauman':['Spenser Andrew Bauman'], + 'Raffael Tfirst':['raffael.tfirst at gmail.com'], + 'timo':['timo at eistee.fritz.box'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,145 +1,7 @@ ========================= -What's new in PyPy 5.1+ +What's new in PyPy2.7 5.3+ ========================= -.. this is a revision shortly after release-5.1 -.. startrev: aa60332382a1 +.. this is a revision shortly after release-pypy2.7-v5.3 +.. startrev: 873218a739f1 -.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046 - -.. branch: gcheader-decl - -Reduce the size of generated C sources. - - -.. branch: remove-objspace-options - -Remove a number of options from the build process that were never tested and -never set. Fix a performance bug in the method cache. - -.. branch: bitstring - -JIT: use bitstrings to compress the lists of read or written descrs -that we attach to EffectInfo. Fixes a problem we had in -remove-objspace-options. - -.. branch: cpyext-for-merge - -Update cpyext C-API support After this branch, we are almost able to support -upstream numpy via cpyext, so we created (yet another) fork of numpy at -github.com/pypy/numpy with the needed changes. Among the significant changes -to cpyext: - - allow c-snippet tests to be run with -A so we can verify we are compatible - - fix many edge cases exposed by fixing tests to run with -A - - issequence() logic matches cpython - - make PyStringObject and PyUnicodeObject field names compatible with cpython - - add prelminary support for PyDateTime_* - - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, - PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, - - PyAnySet_CheckExact, PyUnicode_Concat - - improve support for PyGILState_Ensure, PyGILState_Release, and thread - primitives, also find a case where CPython will allow thread creation - before PyEval_InitThreads is run, dissallow on PyPy - - create a PyObject-specific list strategy - - rewrite slot assignment for typeobjects - - improve tracking of PyObject to rpython object mapping - - support tp_as_{number, sequence, mapping, buffer} slots - -(makes the pypy-c bigger; this was fixed subsequently by the -share-cpyext-cpython-api branch) - -.. branch: share-mapdict-methods-2 - -Reduce generated code for subclasses by using the same function objects in all -generated subclasses. - -.. branch: share-cpyext-cpython-api - -.. branch: cpyext-auto-gil - -CPyExt tweak: instead of "GIL not held when a CPython C extension module -calls PyXxx", we now silently acquire/release the GIL. Helps with -CPython C extension modules that call some PyXxx() functions without -holding the GIL (arguably, they are theorically buggy). - -.. branch: cpyext-test-A - -Get the cpyext tests to pass with "-A" (i.e. when tested directly with -CPython). - -.. branch: oefmt - -.. branch: cpyext-werror - -Compile c snippets with -Werror in cpyext - -.. branch: gc-del-3 - -Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. -It is a more flexible way to make RPython finalizers. - -.. branch: unpacking-cpython-shortcut - -.. branch: cleanups - -.. branch: cpyext-more-slots - -.. branch: use-gc-del-3 - -Use the new rgc.FinalizerQueue mechanism to clean up the handling of -``__del__`` methods. Fixes notably issue #2287. (All RPython -subclasses of W_Root need to use FinalizerQueue now.) - -.. branch: ufunc-outer - -Implement ufunc.outer on numpypy - -.. branch: verbose-imports - -Support ``pypy -v``: verbose imports. It does not log as much as -cpython, but it should be enough to help when debugging package layout -problems. - -.. branch: cpyext-macros-cast - -Fix some warnings when compiling CPython C extension modules - -.. branch: syntax_fix - -.. branch: remove-raisingops - -Remove most of the _ovf, _zer and _val operations from RPython. Kills -quite some code internally, and allows the JIT to do better -optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` -can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly -negative. - -.. branch: cpyext-old-buffers - -Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap - -.. branch: numpy-includes - -Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy -This allows building upstream numpy and scipy in pypy via cpyext - -.. branch: traceviewer-common-merge-point-formats - -Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. - -.. branch: cpyext-pickle - -Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch -at cpyext import time - -.. branch: nonmovable-list - -Add a way to ask "give me a raw pointer to this list's -items". Only for resizable lists of primitives. Turns the GcArray -nonmovable, possibly making a copy of it first. - -.. branch: cpyext-ext - -Finish the work already partially merged in cpyext-for-merge. Adds support -for ByteArrayObject using the nonmovable-list, which also enables -buffer(bytearray()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -1,5 +1,5 @@ ========================= -What's new in PyPy 5.1+ +What's new in PyPy2.7 5.3 ========================= .. this is a revision shortly after release-5.1 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.6.0" +VERSION = "1.7.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.6.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.7.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1359,8 +1359,8 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("enum foo;") from cffi import __version_info__ - if __version_info__ < (1, 7): - py.test.skip("re-enable me in version 1.7") + if __version_info__ < (1, 8): + py.test.skip("re-enable me in version 1.8") e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) assert str(e.value) == ( "'enum foo' has no values explicitly defined: refusing to guess " diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -1909,3 +1909,10 @@ assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'], ['CFFIa', 'CFFIcc', 'CFFIccc'], ['CFFIaa', 'CFFIaaa', 'CFFIg']) + +def test_bool_in_cpp(): + # this works when compiled as C, but in cffi < 1.7 it fails as C++ + ffi = FFI() + ffi.cdef("bool f(void);") + lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }") + assert lib.f() == 1 diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,7 +1,7 @@ # Edit these appropriately before running this script maj=5 -min=1 -rev=2 +min=3 +rev=0 branchname=release-$maj.x # ==OR== release-$maj.$min.x tagname=release-$maj.$min.$rev # ==OR== release-$maj.$min @@ -10,6 +10,7 @@ echo checking hg log -r $tagname hg log -r $tagname || exit 1 +rel=pypy2-v$maj.$min.$rev # This script will download latest builds from the buildmaster, rename the top # level directory, and repackage ready to be uploaded to bitbucket. It will also # download source, assuming a tag for the release already exists, and repackage them. @@ -24,27 +25,27 @@ if [ $plat = linux ]; then plat_final=linux32 fi - mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat_final + mv pypy-c-jit-*-$plat $rel-$plat_final echo packaging $plat_final - tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-$plat_final.tar.bz2 pypy-$maj.$min.$rev-$plat_final - rm -rf pypy-$maj.$min.$rev-$plat_final + tar --owner=root --group=root --numeric-owner -cjf $rel-$plat_final.tar.bz2 $rel-$plat_final + rm -rf $rel-$plat_final done plat=win32 wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip unzip pypy-c-jit-latest-$plat.zip -mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat -zip -r pypy-$maj.$min.$rev-$plat.zip pypy-$maj.$min.$rev-$plat -rm -rf pypy-$maj.$min.$rev-$plat +mv pypy-c-jit-*-$plat $rel-$plat +zip -r $rel-$plat.zip $rel-$plat +rm -rf $rel-$plat # Do this after creating a tag, note the untarred directory is pypy-pypy- # so make sure there is not another one wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2 tar -xf $tagname.tar.bz2 -mv pypy-pypy-* pypy-$maj.$min.$rev-src -tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-src.tar.bz2 pypy-$maj.$min.$rev-src -zip -r pypy-$maj.$min.$rev-src.zip pypy-$maj.$min.$rev-src -rm -rf pypy-$maj.$min.$rev-src +mv pypy-pypy-* $rel-src +tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src +zip -r $rel-src.zip $rel-src +rm -rf $rel-src # Print out the md5, sha1, sha256 md5sum *.bz2 *.zip diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -398,8 +398,8 @@ self.rm.force_spill_var(var) def before_call(self, save_all_regs=False): - self.rm.before_call(save_all_regs) - self.vfprm.before_call(save_all_regs) + self.rm.before_call(save_all_regs=save_all_regs) + self.vfprm.before_call(save_all_regs=save_all_regs) def _sync_var(self, v): if v.type == FLOAT: diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -1241,6 +1241,7 @@ return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list, v_list) + at jit.dont_look_inside def ll_nonmovable_raw_ptr_for_resizable_list(ll_list): """ WARNING: dragons ahead. diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -204,6 +204,9 @@ blockcount = 0 self.annmixlevel = None while True: + # make sure all reprs so far have had their setup() called + self.call_all_setups() + # look for blocks not specialized yet pending = [block for block in self.annotator.annotated if block not in self.already_seen] @@ -236,8 +239,6 @@ previous_percentage = percentage self.log.event('specializing: %d / %d blocks (%d%%)' % (n, total, percentage)) - # make sure all reprs so far have had their setup() called - self.call_all_setups() self.log.event('-=- specialized %d%s blocks -=-' % ( blockcount, newtext)) diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -48,7 +48,6 @@ self.delayedfunctionptrs = [] self.completedcontainers = 0 self.containerstats = {} - self.helpers = OrderedDict() # late_initializations is for when the value you want to # assign to a constant object is something C doesn't think is diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -7,27 +7,6 @@ from rpython.translator.c.support import cdecl -def predeclare_common_types(db, rtyper): - # Common types - yield ('RPyString', STR) - -def predeclare_utility_functions(db, rtyper): - # Common utility functions - def RPyString_New(length=lltype.Signed): - return mallocstr(length) - - for fname, f in locals().items(): - if isinstance(f, types.FunctionType): - # XXX this is painful :( - if fname in db.helpers: - yield (fname, db.helpers[fname]) - else: - # hack: the defaults give the type of the arguments - graph = rtyper.annotate_helper(f, f.func_defaults) - db.helpers[fname] = graph - yield (fname, graph) - - def predeclare_exception_data(exctransformer, rtyper): # Exception-related types and constants exceptiondata = rtyper.exceptiondata @@ -56,11 +35,8 @@ def predeclare_all(db, rtyper): - for fn in [predeclare_common_types, - predeclare_utility_functions, - ]: - for t in fn(db, rtyper): - yield t + # Common types + yield ('RPyString', STR) exctransformer = db.exctransformer for t in predeclare_exception_data(exctransformer, rtyper): diff --git a/rpython/translator/c/src/rtyper.c b/rpython/translator/c/src/rtyper.c --- a/rpython/translator/c/src/rtyper.c +++ b/rpython/translator/c/src/rtyper.c @@ -36,11 +36,3 @@ free(dump); } } - -RPyString *RPyString_FromString(char *buf) -{ - int length = strlen(buf); - RPyString *rps = RPyString_New(length); - memcpy(rps->rs_chars.items, buf, length); - return rps; -} diff --git a/rpython/translator/c/src/rtyper.h b/rpython/translator/c/src/rtyper.h --- a/rpython/translator/c/src/rtyper.h +++ b/rpython/translator/c/src/rtyper.h @@ -11,4 +11,3 @@ RPY_EXTERN char *RPyString_AsCharP(RPyString *rps); RPY_EXTERN void RPyString_FreeCache(void); -RPY_EXTERN RPyString *RPyString_FromString(char *buf); From pypy.commits at gmail.com Mon Jun 6 06:27:31 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 03:27:31 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress: replay mode Message-ID: <57555013.e153c20a.64238.ffffcf8d@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84963:db02f646172a Date: 2016-06-06 12:28 +0200 http://bitbucket.org/pypy/pypy/changeset/db02f646172a/ Log: in-progress: replay mode diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -400,7 +400,7 @@ break elif self.db.reverse_debugger: from rpython.translator.revdb import revdb_genc - line += '\n' + revdb_genc.emit(self.lltypename(v_result), r) + line = revdb_genc.emit(line, self.lltypename(v_result), r) return line def OP_DIRECT_CALL(self, op): @@ -440,8 +440,8 @@ S = self.lltypemap(op.args[0]).TO if S._gckind != 'gc' and not S._hints.get('is_excdata'): from rpython.translator.revdb import revdb_genc - result += '\t' + revdb_genc.emit(self.lltypename(op.result), - newvalue) + result = revdb_genc.emit(result, self.lltypename(op.result), + newvalue) return result def generic_set(self, op, targetexpr): diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -37,6 +37,10 @@ # include #endif +#ifdef RPY_REVERSE_DEBUGGER +# include +#endif + RPY_EXPORTED void rpython_startup_code(void) { @@ -83,7 +87,7 @@ instrument_setup(); #ifdef RPY_REVERSE_DEBUGGER - rpy_reverse_db_setup(argc, argv); + rpy_reverse_db_setup(&argc, &argv); #endif #ifndef MS_WINDOWS @@ -99,6 +103,10 @@ exitcode = STANDALONE_ENTRY_POINT(argc, argv); +#ifdef RPY_REVERSE_DEBUGGER + rpy_reverse_db_teardown(); +#endif + pypy_debug_alloc_results(); if (RPyExceptionOccurred()) { diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -187,9 +187,9 @@ rules = [ ('all', '$(DEFAULT_TARGET)', []), ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'), - ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), - ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), - ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), + ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'), + ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'), + ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'), ] for rule in rules: diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -15,12 +15,18 @@ static int rpy_rev_fileno = -1; +/* ------------------------------------------------------------ */ +#ifndef RPY_RDB_REPLAY +/* ------------------------------------------------------------ */ + + RPY_EXTERN -void rpy_reverse_db_setup(int argc, char *argv[]) +void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) { /* init-time setup */ char *filename = getenv("PYPYREVDB"); + Signed x; rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; @@ -37,12 +43,18 @@ atexit(rpy_reverse_db_flush); } - rpy_reverse_db_EMIT(Signed _e=RDB_SIGNATURE); - rpy_reverse_db_EMIT(Signed _e=RDB_VERSION); - rpy_reverse_db_EMIT(Signed _e=0); - rpy_reverse_db_EMIT(Signed _e=0); - rpy_reverse_db_EMIT(Signed _e=argc); - rpy_reverse_db_EMIT(void *_e=argv); + RPY_REVDB_EMIT(x = RDB_SIGNATURE;, Signed _e, x); + RPY_REVDB_EMIT(x = RDB_VERSION;, Signed _e, x); + RPY_REVDB_EMIT(x = 0;, Signed _e, x); + RPY_REVDB_EMIT(x = 0;, Signed _e, x); + RPY_REVDB_EMIT(x = *argc_p;, Signed _e, x); + RPY_REVDB_EMIT(x = (Signed)*argv_p;, Signed _e, x); +} + +RPY_EXTERN +void rpy_reverse_db_teardown(void) +{ + rpy_reverse_db_flush(); } RPY_EXTERN @@ -59,3 +71,97 @@ } } } + + +/* ------------------------------------------------------------ */ +#else +/* ------------------------------------------------------------ */ + + +RPY_EXTERN +void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) +{ + Signed x; + + if (*argc_p <= 1) { + rpy_rev_fileno = 0; /* stdin */ + } + else { + char *filename = (*argv_p)[1]; + rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY); + if (rpy_rev_fileno < 0) { + fprintf(stderr, "Can't open file '%s': %m\n", filename); + exit(1); + } + } + rpy_revdb.buf_p = rpy_rev_buffer; + rpy_revdb.buf_limit = rpy_rev_buffer; + + RPY_REVDB_EMIT(*, Signed _e, x); + if (x != RDB_SIGNATURE) { + fprintf(stderr, "stdin is not a RevDB file (or wrong platform)\n"); + exit(1); + } + RPY_REVDB_EMIT(*, Signed _e, x); + if (x != RDB_VERSION) { + fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n", + (long)x, (long)RDB_VERSION); + exit(1); + } + RPY_REVDB_EMIT(*, Signed _e, x); /* ignored */ + RPY_REVDB_EMIT(*, Signed _e, x); /* ignored */ + + RPY_REVDB_EMIT(*, Signed _e, x); + if (x <= 0) { + fprintf(stderr, "RevDB file is bogus\n"); + exit(1); + } + *argc_p = x; + + RPY_REVDB_EMIT(*, Signed _e, x); + *argv_p = (char **)x; +} + +RPY_EXTERN +void rpy_reverse_db_teardown(void) +{ + char dummy[1]; + if (rpy_revdb.buf_p != rpy_revdb.buf_limit || + read(rpy_rev_fileno, dummy, 1) > 0) { + fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); + exit(1); + } +} + +RPY_EXTERN +char *rpy_reverse_db_fetch(int expected_size) +{ + ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p; + assert(keep >= 0); + memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep); + + retry: + rsize = read(rpy_rev_fileno, rpy_rev_buffer + keep, + sizeof(rpy_rev_buffer) - keep); + if (rsize <= 0) { + if (rsize == 0) + fprintf(stderr, "RevDB file appears truncated\n"); + else + fprintf(stderr, "RevDB file read error: %m\n"); + exit(1); + } + keep += rsize; + + rpy_revdb.buf_p = rpy_rev_buffer; + rpy_revdb.buf_limit = rpy_rev_buffer + keep; + + if (rpy_revdb.buf_limit - rpy_revdb.buf_p < expected_size) + goto retry; + + return rpy_rev_buffer; +} + + +/* ------------------------------------------------------------ */ +#endif +/* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/revdb/rdb-src/revdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -1,15 +1,57 @@ #include -RPY_EXTERN void rpy_reverse_db_setup(int argc, char *argv[]); -RPY_EXTERN void rpy_reverse_db_flush(void); - +RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); +RPY_EXTERN void rpy_reverse_db_teardown(void); typedef struct { char *buf_p, *buf_limit; } rpy_revdb_t; RPY_EXTERN rpy_revdb_t rpy_revdb; -#define rpy_reverse_db_EMIT(decl_e) do { \ - decl_e; \ - memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ - if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ - rpy_reverse_db_flush(); \ -} while (0) + +/* ------------------------------------------------------------ */ +#ifndef RPY_RDB_REPLAY +/* ------------------------------------------------------------ */ + + +/* recording version of the macros */ +#define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ + normal_code \ + do { \ + decl_e = variable; \ + memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ + if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ + rpy_reverse_db_flush(); \ + } while (0) +#define RPY_REVDB_EMIT_VOID(normal_code) \ + normal_code + +RPY_EXTERN void rpy_reverse_db_flush(void); + + +/* ------------------------------------------------------------ */ +#else +/* ------------------------------------------------------------ */ + + +/* replaying version of the macros */ +#define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ + do { \ + decl_e; \ + char *_src = rpy_revdb.buf_p; \ + char *_end1 = _src + sizeof(_e); \ + if (_end1 > rpy_revdb.buf_limit) { \ + _src = rpy_reverse_db_fetch(sizeof(_e)); \ + _end1 = _src + sizeof(_e); \ + } \ + rpy_revdb.buf_p = _end1; \ + memcpy(&_e, _src, sizeof(_e)); \ + variable = _e; \ + } while (0) +#define RPY_REVDB_EMIT_VOID(normal_code) \ + /* nothing */ + +RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size); + + +/* ------------------------------------------------------------ */ +#endif +/* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py --- a/rpython/translator/revdb/revdb_genc.py +++ b/rpython/translator/revdb/revdb_genc.py @@ -9,7 +9,7 @@ srcdir / 'revdb.c', ] -def emit(tp, value): +def emit(normal_code, tp, value): if tp == 'void @': - return '/* void */' - return 'rpy_reverse_db_EMIT(%s=%s);' % (cdecl(tp, '_e'), value) + return 'RPY_REVDB_EMIT_VOID(%s);' % (normal_code,) + return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value) diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -86,3 +86,6 @@ assert rdb.done() # assert got == [self.exename, 'abc', 'd'] + # + # Now try the replay mode + xxx From pypy.commits at gmail.com Mon Jun 6 08:52:39 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 05:52:39 -0700 (PDT) Subject: [pypy-commit] cffi default: Write down a typical ffi.new_handle() usage example Message-ID: <57557217.4674c20a.8a463.2206@mx.google.com> Author: Armin Rigo Branch: Changeset: r2712:e2f143d0e9b4 Date: 2016-06-06 14:53 +0200 http://bitbucket.org/cffi/cffi/changeset/e2f143d0e9b4/ Log: Write down a typical ffi.new_handle() usage example diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -359,6 +359,8 @@ *Calling ffi.from_handle(p) is invalid and will likely crash if the cdata object returned by new_handle() is not kept alive!* +See a `typical usage example`_ below. + (In case you are wondering, this ``void *`` is not the ``PyObject *`` pointer. This wouldn't make sense on PyPy anyway.) @@ -391,6 +393,33 @@ ``global_set.discard(p)``, with ``p`` any cdata object whose ``void *`` value compares equal. +.. _`typical usage example`: + +Usage example: suppose you have a C library where you must call a +``lib.process_document()`` function which invokes some callback. The +``process_document()`` function receives a pointer to a callback and a +``void *`` argument. The callback is then invoked with the ``void +*data`` argument that is equal to the provided value. In this typical +case, you can implement it like this (out-of-line API mode):: + + class MyDocument: + ... + + def process(self): + lib.process_document(lib.my_callback, # the callback + ffi.new_handle(self), # 'void *data' + args...) + # 'self' stay alive at least until here, which means that + # the ffi.from_handle() done in my_callback() are safe + + def callback(self, arg1, arg2): + ... + + # the actual callback is this one-liner global function: + @ffi.def_extern + def my_callback(arg1, arg2, data): + return ffi.from_handle(data).callback(arg1, arg2) + .. _ffi-dlopen: .. _ffi-dlclose: From pypy.commits at gmail.com Mon Jun 6 10:20:06 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 07:20:06 -0700 (PDT) Subject: [pypy-commit] pypy default: link to the whatsnew of cffi 1.7 Message-ID: <57558696.a508c30a.eddbe.2054@mx.google.com> Author: Armin Rigo Branch: Changeset: r84964:503409ea9765 Date: 2016-06-06 16:20 +0200 http://bitbucket.org/pypy/pypy/changeset/503409ea9765/ Log: link to the whatsnew of cffi 1.7 diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -11,7 +11,9 @@ We also improved the speed of ... and ... -We updated cffi_ to ... +We updated cffi_ to version 1.7 (small changes, documented here__). + +.. __: http://cffi.readthedocs.io/en/latest/whatsnew.html You can download the PyPy2.7 v5.3 release here: From pypy.commits at gmail.com Mon Jun 6 10:23:45 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 07:23:45 -0700 (PDT) Subject: [pypy-commit] cffi default: Update Message-ID: <57558771.10301c0a.e92b1.0519@mx.google.com> Author: Armin Rigo Branch: Changeset: r2713:d02a2bd8752d Date: 2016-06-06 16:25 +0200 http://bitbucket.org/cffi/cffi/changeset/d02a2bd8752d/ Log: Update diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -16,13 +16,13 @@ * bytearrays: ``ffi.from_buffer(bytearray-object)`` is now supported. (The reason it was not supported was that it was hard to do in PyPy, - but it works since PyPy 5.2.) You can call C functions with a - ``char *`` argument from any buffer object, now including - bytearrays, by giving them ``ffi.from_buffer(..)`` as argument. - Additionally, you can now do ``p[0:length] = bytearray-object``. - The problem with this was that a iterating over bytearrays gives - *numbers* instead of *characters*. (Now it is implemented with just - a memcpy, of course, not actually iterating over the characters.) + but it works since PyPy 5.3.) To call a C function with a ``char *`` + argument from a buffer object---now including bytearrays---you write + ``lib.foo(ffi.from_buffer(x))``. Additionally, this is now supported: + ``p[0:length] = bytearray-object``. The problem with this was that a + iterating over bytearrays gives *numbers* instead of *characters*. + (Now it is implemented with just a memcpy, of course, not actually + iterating over the characters.) * C++: compiling the generated C code with C++ was supposed to work, but failed if you make use the ``bool`` type (because that is rendered From pypy.commits at gmail.com Mon Jun 6 10:27:17 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 07:27:17 -0700 (PDT) Subject: [pypy-commit] pypy default: Kill this other mention of cffi Message-ID: <57558845.094ac20a.10b2f.3f94@mx.google.com> Author: Armin Rigo Branch: Changeset: r84965:5acb6054c9c5 Date: 2016-06-06 16:27 +0200 http://bitbucket.org/pypy/pypy/changeset/5acb6054c9c5/ Log: Kill this other mention of cffi diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -109,12 +109,10 @@ * Revive traceviewer, a tool to use pygame to view traces - * Update to cffi/847bbc0297f8 which improves help() on cffi objects - * Bug Fixes - * Fix issue #2277: only special-case two exact lists in zip(), not list - subclasses, because an overridden __iter__() should be called (probably) + * Fix issue #2277: only special-case two exact lists in zip(), not list + subclasses, because an overridden __iter__() should be called (probably) * Fix issue #2226: Another tweak in the incremental GC- this should ensure that progress in the major GC occurs quickly enough in all cases. From pypy.commits at gmail.com Mon Jun 6 10:30:55 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 07:30:55 -0700 (PDT) Subject: [pypy-commit] cffi default: Another new item Message-ID: <5755891f.879d1c0a.68aaa.6369@mx.google.com> Author: Armin Rigo Branch: Changeset: r2714:de7a9358afd0 Date: 2016-06-06 16:32 +0200 http://bitbucket.org/cffi/cffi/changeset/de7a9358afd0/ Log: Another new item diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -28,6 +28,8 @@ but failed if you make use the ``bool`` type (because that is rendered as the C ``_Bool`` type, which doesn't exist in C++). +* ``help(lib)`` and ``help(lib.myfunc)`` now give useful information. + v1.6 ==== From pypy.commits at gmail.com Mon Jun 6 10:48:01 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 07:48:01 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove unnecessary conftest imports Message-ID: <57558d21.8b371c0a.ae99c.002d@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84966:9f49da4855da Date: 2016-06-06 15:47 +0100 http://bitbucket.org/pypy/pypy/changeset/9f49da4855da/ Log: Remove unnecessary conftest imports diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,5 +1,5 @@ +import pytest from rpython.tool import udir -from pypy.conftest import option from pypy.interpreter.gateway import interp2app def check_no_w_locals(space, w_frame): @@ -11,7 +11,7 @@ space = cls.space cls.w_udir = cls.space.wrap(str(udir.udir)) cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) - if not option.runappdirect: + if not pytest.config.option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): return f() @@ -48,10 +48,10 @@ return f.f_code assert g() is g.func_code - def test_f_trace_del(self): + def test_f_trace_del(self): import sys - f = sys._getframe() - del f.f_trace + f = sys._getframe() + del f.f_trace assert f.f_trace is None def test_f_lineno(self): @@ -116,7 +116,7 @@ def f(): assert sys._getframe().f_code.co_name == g() def g(): - return sys._getframe().f_back.f_code.co_name + return sys._getframe().f_back.f_code.co_name f() def test_f_back_virtualref(self): @@ -233,7 +233,7 @@ def test_trace_exc(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -298,7 +298,7 @@ def test_trace_return_exc(self): import sys l = [] - def trace(a,b,c): + def trace(a,b,c): if b in ('exception', 'return'): l.append((b, c)) return trace @@ -444,7 +444,7 @@ def test_dont_trace_on_reraise(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -466,7 +466,7 @@ def test_dont_trace_on_raise_with_tb(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -1,9 +1,8 @@ import py -from pypy import conftest from pypy.interpreter import gateway from rpython.rlib.jit import non_virtual_ref, vref_None -class AppTestSlow: +class AppTestSlow: spaceconfig = dict(usemodules=['itertools']) def setup_class(cls): @@ -64,7 +63,7 @@ space.setitem(space.builtin.w_dict, space.wrap('read_exc_type'), space.wrap(read_exc_type_gw)) - + def _detach_helpers(space): space.delitem(space.builtin.w_dict, space.wrap('hide_top_frame')) @@ -92,7 +91,7 @@ pckl = pickle.dumps(code) result = pickle.loads(pckl) assert code == result - + def test_pickle_global_func(self): import new mod = new.module('mod') @@ -109,7 +108,7 @@ assert func is result finally: del sys.modules['mod'] - + def test_pickle_not_imported_module(self): import new mod = new.module('mod') @@ -119,13 +118,13 @@ result = pickle.loads(pckl) assert mod.__name__ == result.__name__ assert mod.__dict__ == result.__dict__ - + def test_pickle_builtin_func(self): import pickle pckl = pickle.dumps(map) result = pickle.loads(pckl) assert map is result - + def test_pickle_non_top_reachable_func(self): def func(): return 42 @@ -142,7 +141,7 @@ assert func.func_dict == result.func_dict assert func.func_doc == result.func_doc assert func.func_globals == result.func_globals - + def test_pickle_cell(self): def g(): x = [42] @@ -171,7 +170,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) assert type(f1) is type(f2) @@ -223,7 +222,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) def test_frame_setstate_crash(self): @@ -257,21 +256,21 @@ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_moduledict(self): import pickle moddict = pickle.__dict__ pckl = pickle.dumps(moddict) result = pickle.loads(pckl) assert moddict is result - + def test_pickle_bltins_module(self): import pickle mod = __builtins__ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_buffer(self): skip("Can't pickle buffer objects on top of CPython either. " "Do we really need it?") @@ -280,14 +279,14 @@ pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_complex(self): import pickle a = complex(1.23,4.567) pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_method(self): class myclass(object): def f(self): @@ -308,7 +307,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_staticmethod(self): class myclass(object): def f(): @@ -319,7 +318,7 @@ pckl = pickle.dumps(method) result = pickle.loads(pckl) assert method() == result() - + def test_pickle_classmethod(self): class myclass(object): def f(cls): @@ -337,7 +336,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_sequenceiter(self): ''' In PyPy there is no distinction here between listiterator and diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -89,8 +89,7 @@ class AppTestWrapper: def setup_class(cls): "Run test_various_depths() when we are run with 'pypy py.test -A'." - from pypy.conftest import option - if not option.runappdirect: + if not py.test.config.option.runappdirect: py.test.skip("meant only for -A run") def _setup(): diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -516,12 +516,11 @@ assert s == 'bar\n' def test_flush_at_exit(): - from pypy import conftest from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) + config = make_config(py.test.config.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): f = open(tmpfile, 'w') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -222,14 +222,10 @@ assert not closed[0] # flush() called before file closed os.close(fd) -def test_flush_at_exit(): - from pypy import conftest - from pypy.tool.option import make_config, make_objspace +def test_flush_at_exit(space): from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) - space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): import io f = io.open(tmpfile, 'w', encoding='ascii') @@ -241,12 +237,7 @@ assert tmpfile.read() == '42' -def test_flush_at_exit_IOError_and_ValueError(): - from pypy import conftest - from pypy.tool.option import make_config, make_objspace - - config = make_config(conftest.option) - space = make_objspace(config) +def test_flush_at_exit_IOError_and_ValueError(space): space.appexec([], """(): import io class MyStream(io.IOBase): diff --git a/pypy/module/_rawffi/test/test_tracker.py b/pypy/module/_rawffi/test/test_tracker.py --- a/pypy/module/_rawffi/test/test_tracker.py +++ b/pypy/module/_rawffi/test/test_tracker.py @@ -1,5 +1,5 @@ import py -from pypy.conftest import option +from pytest import option from pypy.module._rawffi.tracker import Tracker @@ -44,4 +44,3 @@ def teardown_class(cls): Tracker.DO_TRACING = False - diff --git a/pypy/module/gc/test/test_referents.py b/pypy/module/gc/test/test_referents.py --- a/pypy/module/gc/test/test_referents.py +++ b/pypy/module/gc/test/test_referents.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from pytest import option class AppTestReferents(object): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -13,7 +13,7 @@ from pypy.module.imp import importing -from pypy import conftest +from pytest import config def setuppkg(pkgname, **entries): p = udir.join('impsubdir') @@ -106,7 +106,7 @@ # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") - if conftest.option.runappdirect: + if config.option.runappdirect: import marshal, stat, struct, os, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) @@ -143,7 +143,7 @@ def _setup(space): dn = setup_directory_structure(space) return space.appexec([space.wrap(dn)], """ - (dn): + (dn): import sys path = list(sys.path) sys.path.insert(0, dn) @@ -168,7 +168,7 @@ } def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -1017,7 +1017,7 @@ cpathname = udir.join('test.pyc') assert not cpathname.check() - + def test_load_source_module_importerror(self): # the .pyc file is created before executing the module space = self.space @@ -1126,11 +1126,11 @@ stream.close() -def test_PYTHONPATH_takes_precedence(space): +def test_PYTHONPATH_takes_precedence(space): if sys.platform == "win32": py.test.skip("unresolved issues with win32 shell quoting rules") - from pypy.interpreter.test.test_zpy import pypypath - extrapath = udir.ensure("pythonpath", dir=1) + from pypy.interpreter.test.test_zpy import pypypath + extrapath = udir.ensure("pythonpath", dir=1) extrapath.join("sched.py").write("print 42\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) @@ -1494,8 +1494,6 @@ spaceconfig = dict(usemodules=['thread', 'time']) def setup_class(cls): - #if not conftest.option.runappdirect: - # py.test.skip("meant as an -A test") tmpfile = udir.join('test_multithreaded_imp.py') tmpfile.write('''if 1: x = 666 diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy import constants as NPY @@ -7,7 +7,7 @@ @classmethod def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: import sys if '__pypy__' not in sys.builtin_module_names: import numpy diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -2,7 +2,6 @@ import sys -from pypy.conftest import option from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest @@ -23,13 +22,13 @@ # special values testing if isnan(a): if isnan(b): - return True,'' + return True, '' raise AssertionError(msg + '%r should be nan' % (b,)) if isinf(a): if a == b: - return True,'' - raise AssertionError(msg + 'finite result where infinity expected: '+ \ + return True, '' + raise AssertionError(msg + 'finite result where infinity expected: ' 'expected %r, got %r' % (a, b)) # if both a and b are zero, check whether they have the same sign @@ -39,7 +38,7 @@ if not a and not b: # only check it if we are running on top of CPython >= 2.6 if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b): - raise AssertionError( msg + \ + raise AssertionError(msg + 'zero has wrong sign: expected %r, got %r' % (a, b)) # if a-b overflows, or b is infinite, return False. Again, in @@ -96,7 +95,6 @@ cls.w_testcases128 = cls.space.wrap(list(parse_testfile(fname128))) cls.w_testcases64 = cls.space.wrap(list(parse_testfile(fname64))) - cls.w_runAppDirect = cls.space.wrap(option.runappdirect) cls.w_isWindows = cls.space.wrap(os.name == 'nt') if cls.runappdirect: @@ -495,8 +493,8 @@ c = array([1.e+110, 1.e-110], dtype=complex128) d = floor_divide(c**2, c) assert (d == [1.e+110, 0]).all() - - + + def test_basic(self): import sys diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1,11 +1,11 @@ -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.interpreter.gateway import interp2app class BaseAppTestDtypes(BaseNumpyAppTest): def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if option.runappdirect: + if config.option.runappdirect: import platform bits, linkage = platform.architecture() ptr_size = int(bits[:-3]) // 8 @@ -372,8 +372,8 @@ a = np.array(data, dtype=b) x = pickle.loads(pickle.dumps(a)) assert (x == a).all() - assert x.dtype == a.dtype - + assert x.dtype == a.dtype + def test_index(self): import numpy as np for dtype in [np.int8, np.int16, np.int32, np.int64]: @@ -1088,7 +1088,7 @@ spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if option.runappdirect: + if config.option.runappdirect: cls.w_test_for_core_internal = cls.space.wrap(True) else: cls.w_test_for_core_internal = cls.space.wrap(False) @@ -1459,7 +1459,7 @@ "'offsets':[0,76800], " "'itemsize':80000, " "'aligned':True}") - + assert dt == np.dtype(eval(str(dt))) dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'], @@ -1515,7 +1515,7 @@ else: assert stor2[1] == '\x01' assert stor2[0] == '\x00' - if option.runappdirect: + if config.option.runappdirect: cls.w_check_non_native = lambda *args : None else: cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2,7 +2,7 @@ import py import sys -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk from pypy.module.micronumpy.ndarray import W_NDimArray @@ -1878,7 +1878,7 @@ assert map(isnan, e) == [False, False, False, True, False] assert map(isinf, e) == [False, False, True, False, False] assert e.argmax() == 3 - # numpy preserves value for uint16 -> cast_as_float16 -> + # numpy preserves value for uint16 -> cast_as_float16 -> # convert_to_float64 -> convert_to_float16 -> uint16 # even for float16 various float16 nans all_f16 = arange(0xfe00, 0xffff, dtype='uint16') @@ -2608,7 +2608,7 @@ a = np.arange(6).reshape(2,3) i = np.dtype('int32').type(0) assert (a[0] == a[i]).all() - + def test_ellipsis_indexing(self): import numpy as np @@ -3850,7 +3850,7 @@ class AppTestRepr(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: py.test.skip("Can't be run directly.") BaseNumpyAppTest.setup_class.im_func(cls) cache = get_appbridge_cache(cls.space) @@ -3867,7 +3867,7 @@ assert repr(array(1.5).real) == "array(1.5)" def teardown_class(cls): - if option.runappdirect: + if config.option.runappdirect: return cache = get_appbridge_cache(cls.space) cache.w_array_repr = cls.old_array_repr @@ -4343,7 +4343,7 @@ class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if config.option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -1,14 +1,9 @@ from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pypy.conftest import option class AppTestObjectDtypes(BaseNumpyAppTest): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) - def setup_class(cls): - BaseNumpyAppTest.setup_class.im_func(cls) - cls.w_runappdirect = cls.space.wrap(option.runappdirect) - def test_scalar_from_object(self): from numpy import array import sys @@ -200,7 +195,7 @@ from numpy import arange, dtype from cPickle import loads, dumps import sys - + a = arange(15).astype(object) if '__pypy__' in sys.builtin_module_names: raises(NotImplementedError, dumps, a) @@ -211,4 +206,4 @@ a = arange(15).astype(object).reshape((3, 5)) b = loads(dumps(a)) assert (a == b).all() - + diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py --- a/pypy/module/micronumpy/test/test_support_app.py +++ b/pypy/module/micronumpy/test/test_support_app.py @@ -3,11 +3,12 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pypy.conftest import option +from pytest import config class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if (config.option.runappdirect and + '__pypy__' not in sys.builtin_module_names): py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/test_lib_pypy/support.py b/pypy/module/test_lib_pypy/support.py --- a/pypy/module/test_lib_pypy/support.py +++ b/pypy/module/test_lib_pypy/support.py @@ -1,6 +1,6 @@ import py -from pypy.conftest import option +from pytest import config from pypy.interpreter.error import OperationError def import_lib_pypy(space, name, skipmsg=None): @@ -9,7 +9,7 @@ Raises a pytest Skip on ImportError if a skip message was specified. """ - if option.runappdirect: + if config.option.runappdirect: try: mod = __import__('lib_pypy.' + name) except ImportError as e: diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,9 +1,9 @@ -from pypy import conftest +from pytest import config class AppTestBytesArray: def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) def test_basics(self): b = bytearray() diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -1,5 +1,4 @@ -from __future__ import with_statement -from pypy.conftest import option +from pytest import config class AppTestObject: @@ -7,13 +6,13 @@ from pypy.interpreter import gateway import sys - cpython_behavior = (not option.runappdirect + cpython_behavior = (not config.option.runappdirect or not hasattr(sys, 'pypy_translation_info')) space = cls.space cls.w_cpython_behavior = space.wrap(cpython_behavior) cls.w_cpython_version = space.wrap(tuple(sys.version_info)) - cls.w_appdirect = space.wrap(option.runappdirect) + cls.w_appdirect = space.wrap(config.option.runappdirect) cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info')) def w_unwrap_wrap_unicode(space, w_obj): diff --git a/pypy/objspace/std/test/test_proxy_usercreated.py b/pypy/objspace/std/test/test_proxy_usercreated.py --- a/pypy/objspace/std/test/test_proxy_usercreated.py +++ b/pypy/objspace/std/test/test_proxy_usercreated.py @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app from pypy.objspace.std.transparent import register_proxyable -from pypy.conftest import option +from pytest import config class W_Wrapped(W_Root): @@ -25,7 +25,7 @@ class AppTestProxyNewtype(AppProxy): def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: py.test.skip("Impossible to run on appdirect") AppProxy.setup_class.im_func(cls) cls.w_wrapped = cls.space.wrap(W_Wrapped()) diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py --- a/pypy/objspace/test/test_binop_overriding.py +++ b/pypy/objspace/test/test_binop_overriding.py @@ -1,5 +1,5 @@ # test about the binop operation rule, see issue 412 -from pypy.conftest import option +from pytest import config class AppTestBinopCombinations: @@ -73,7 +73,7 @@ if C is not object: setattr(C, name, f) override_in_hier(n-1) - if C is not object: + if C is not object: delattr(C, name) override_in_hier() @@ -83,7 +83,7 @@ return Base, do_test """) cls.w_helpers = w_helpers - cls.w_appdirect = cls.space.wrap(option.runappdirect) + cls.w_appdirect = cls.space.wrap(config.option.runappdirect) def test_overriding_base_binop_explict(self): class MulBase(object): @@ -105,7 +105,7 @@ if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -116,7 +116,7 @@ assert not fail def test_binop_combinations_sub(self): - Base, do_test = self.helpers + Base, do_test = self.helpers class X(Base): pass class Y(X): @@ -124,13 +124,13 @@ fail = do_test(X, Y, 'sub', lambda x,y: x-y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_pow(self): if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -138,13 +138,13 @@ fail = do_test(X, Y, 'pow', lambda x,y: x**y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_more_exhaustive(self): if not self.appdirect: skip("very slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -184,7 +184,6 @@ source = str(source).strip() except py.error.ENOENT: source = None - from pypy import conftest if source and py.test.config._assertstate.mode != "off": msg = interpret(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -3,14 +3,13 @@ from rpython.config.config import ConflictConfigError from pypy.tool.option import make_config, make_objspace from pypy.tool.pytest import appsupport -from pypy.conftest import option _SPACECACHE={} def gettestobjspace(**kwds): """ helper for instantiating and caching space's for testing. """ try: - config = make_config(option,**kwds) + config = make_config(py.test.config.option,**kwds) except ConflictConfigError as e: # this exception is typically only raised if a module is not available. # in this case the test should be skipped @@ -19,7 +18,7 @@ try: return _SPACECACHE[key] except KeyError: - if getattr(option, 'runappdirect', None): + if getattr(py.test.config.option, 'runappdirect', None): return TinyObjSpace(**kwds) space = maketestobjspace(config) _SPACECACHE[key] = space @@ -27,7 +26,7 @@ def maketestobjspace(config=None): if config is None: - config = make_config(option) + config = make_config(py.test.config.option) if config.objspace.usemodules.thread: config.translation.thread = True space = make_objspace(config) From pypy.commits at gmail.com Mon Jun 6 11:18:05 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 08:18:05 -0700 (PDT) Subject: [pypy-commit] pypy default: tweaks Message-ID: <5755942d.879d1c0a.68aaa.78c1@mx.google.com> Author: Matti Picus Branch: Changeset: r84967:901a05dec70d Date: 2016-06-06 18:16 +0300 http://bitbucket.org/pypy/pypy/changeset/901a05dec70d/ Log: tweaks diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -9,11 +9,11 @@ for lxml, we now pass most (more than 90%) of the upstream numpy test suite, and much of SciPy is supported as well. -We also improved the speed of ... and ... +We updated cffi_ to version 1.7 (small changes, documented here_). -We updated cffi_ to version 1.7 (small changes, documented here__). - -.. __: http://cffi.readthedocs.io/en/latest/whatsnew.html +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html +.. _cffi: https://cffi.readthedocs.org +.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html You can download the PyPy2.7 v5.3 release here: @@ -32,10 +32,6 @@ .. _`RPython`: https://rpython.readthedocs.org .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html -.. _`numpy`: https://bitbucket.org/pypy/numpy -.. _cffi: https://cffi.readthedocs.org -.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html -.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html What is PyPy? ============= @@ -44,8 +40,8 @@ CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -We also welcome developers of other -`dynamic languages`_ to see what RPython can do for them. +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. This release supports: @@ -145,7 +141,7 @@ * Implement ufunc.outer on numpypy - * Move PyPy-specific numpy headers to a subdirectory (also changed pypy/numpy + * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_ accordingly) * Performance improvements: @@ -185,9 +181,9 @@ * Compile c snippets with -Werror, and fix warnings it exposed -.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html .. _Numpy: https://bitbucket.org/pypy/numpy +.. _`the repo`: https://bitbucket.org/pypy/numpy Please update, and continue to help us make PyPy better. From pypy.commits at gmail.com Mon Jun 6 12:00:20 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 09:00:20 -0700 (PDT) Subject: [pypy-commit] cffi default: Support dir(p), where p is a struct or pointer-to-struct, to obtain Message-ID: <57559e14.421b1c0a.7265b.ffffa74b@mx.google.com> Author: Armin Rigo Branch: Changeset: r2715:0a5f59abb0e0 Date: 2016-06-06 18:00 +0200 http://bitbucket.org/cffi/cffi/changeset/0a5f59abb0e0/ Log: Support dir(p), where p is a struct or pointer-to-struct, to obtain the list of field names. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2703,6 +2703,27 @@ return res; } +static PyObject *cdata_dir(PyObject *cd, PyObject *noarg) +{ + CTypeDescrObject *ct = ((CDataObject *)cd)->c_type; + + /* replace the type 'pointer-to-t' with just 't' */ + if (ct->ct_flags & CT_POINTER) { + ct = ct->ct_itemdescr; + } + if ((ct->ct_flags & (CT_STRUCT | CT_UNION)) && + !(ct->ct_flags & CT_IS_OPAQUE)) { + + /* for non-opaque structs or unions */ + if (force_lazy_struct(ct) < 0) + return NULL; + return PyDict_Keys(ct->ct_stuff); + } + else { + return PyList_New(0); /* empty list for the other cases */ + } +} + static PyObject *cdata_iter(CDataObject *); static PyNumberMethods CData_as_number = { @@ -2751,6 +2772,11 @@ (objobjargproc)cdata_ass_sub, /*mp_ass_subscript*/ }; +static PyMethodDef cdata_methods[] = { + {"__dir__", cdata_dir, METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + static PyTypeObject CData_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_cffi_backend.CData", @@ -2779,6 +2805,7 @@ offsetof(CDataObject, c_weakreflist), /* tp_weaklistoffset */ (getiterfunc)cdata_iter, /* tp_iter */ 0, /* tp_iternext */ + cdata_methods, /* tp_methods */ }; static PyTypeObject CDataOwning_Type = { diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -88,8 +88,8 @@ assert repr(p) == "" def check_dir(p, expected): - got = set(name for name in dir(p) if not name.startswith('_')) - assert got == set(expected) + got = [name for name in dir(p) if not name.startswith('_')] + assert got == sorted(expected) def test_inspect_primitive_type(): p = new_primitive_type("signed char") @@ -3619,3 +3619,23 @@ # py.test.raises(ValueError, unpack, p0, -1) py.test.raises(ValueError, unpack, p, -1) + +def test_cdata_dir(): + BInt = new_primitive_type("int") + p = cast(BInt, 42) + check_dir(p, []) + p = newp(new_array_type(new_pointer_type(BInt), None), 5) + check_dir(p, []) + BStruct = new_struct_type("foo") + p = cast(new_pointer_type(BStruct), 0) + check_dir(p, []) # opaque + complete_struct_or_union(BStruct, [('a2', BInt, -1), + ('a1', BInt, -1)]) + check_dir(p, ['a1', 'a2']) # always sorted + p = newp(new_pointer_type(BStruct), None) + check_dir(p, ['a1', 'a2']) + check_dir(p[0], ['a1', 'a2']) + pp = newp(new_pointer_type(new_pointer_type(BStruct)), p) + check_dir(pp, []) + check_dir(pp[0], ['a1', 'a2']) + check_dir(pp[0][0], ['a1', 'a2']) diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -28,7 +28,8 @@ but failed if you make use the ``bool`` type (because that is rendered as the C ``_Bool`` type, which doesn't exist in C++). -* ``help(lib)`` and ``help(lib.myfunc)`` now give useful information. +* ``help(lib)`` and ``help(lib.myfunc)`` now give useful information, + as well as ``dir(p)`` where ``p`` is a struct or pointer-to-struct. v1.6 From pypy.commits at gmail.com Mon Jun 6 12:04:19 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 09:04:19 -0700 (PDT) Subject: [pypy-commit] pypy default: update to cffi/0a5f59abb0e0 Message-ID: <57559f03.089d1c0a.5c905.3039@mx.google.com> Author: Armin Rigo Branch: Changeset: r84968:d1760e27726c Date: 2016-06-06 18:03 +0200 http://bitbucket.org/pypy/pypy/changeset/d1760e27726c/ Log: update to cffi/0a5f59abb0e0 diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -420,6 +420,14 @@ w_result = ctype.ctitem.unpack_ptr(ctype, ptr, length) return w_result + def dir(self, space): + from pypy.module._cffi_backend.ctypeptr import W_CTypePointer + ct = self.ctype + if isinstance(ct, W_CTypePointer): + ct = ct.ctitem + lst = ct.cdata_dir() + return space.newlist([space.wrap(s) for s in lst]) + class W_CDataMem(W_CData): """This is used only by the results of cffi.cast('int', x) @@ -602,5 +610,6 @@ __call__ = interp2app(W_CData.call), __iter__ = interp2app(W_CData.iter), __weakref__ = make_weakref_descr(W_CData), + __dir__ = interp2app(W_CData.dir), ) W_CData.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -256,6 +256,9 @@ def fget_elements(self, space): return self._fget('e') def fget_relements(self, space):return self._fget('R') + def cdata_dir(self): + return [] + W_CType.typedef = TypeDef( '_cffi_backend.CTypeDescr', diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -171,6 +171,12 @@ pass return W_CType.getcfield(self, attr) + def cdata_dir(self): + if self.size < 0: + return [] + self.force_lazy_struct() + return self._fields_dict.keys() + class W_CTypeStruct(W_CTypeStructOrUnion): kind = "struct" diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -77,8 +77,8 @@ assert repr(p) == "" def check_dir(p, expected): - got = set(name for name in dir(p) if not name.startswith('_')) - assert got == set(expected) + got = [name for name in dir(p) if not name.startswith('_')] + assert got == sorted(expected) def test_inspect_primitive_type(): p = new_primitive_type("signed char") @@ -3608,3 +3608,23 @@ # py.test.raises(ValueError, unpack, p0, -1) py.test.raises(ValueError, unpack, p, -1) + +def test_cdata_dir(): + BInt = new_primitive_type("int") + p = cast(BInt, 42) + check_dir(p, []) + p = newp(new_array_type(new_pointer_type(BInt), None), 5) + check_dir(p, []) + BStruct = new_struct_type("foo") + p = cast(new_pointer_type(BStruct), 0) + check_dir(p, []) # opaque + complete_struct_or_union(BStruct, [('a2', BInt, -1), + ('a1', BInt, -1)]) + check_dir(p, ['a1', 'a2']) # always sorted + p = newp(new_pointer_type(BStruct), None) + check_dir(p, ['a1', 'a2']) + check_dir(p[0], ['a1', 'a2']) + pp = newp(new_pointer_type(new_pointer_type(BStruct)), p) + check_dir(pp, []) + check_dir(pp[0], ['a1', 'a2']) + check_dir(pp[0][0], ['a1', 'a2']) From pypy.commits at gmail.com Mon Jun 6 12:08:45 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 06 Jun 2016 09:08:45 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix keyword not iterable bug, call walkabout methods instead of visit_sequence, emit ops on call Message-ID: <5755a00d.4674c20a.8a463.77fd@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84969:3cebb79b83a9 Date: 2016-06-06 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/3cebb79b83a9/ Log: fix keyword not iterable bug, call walkabout methods instead of visit_sequence, emit ops on call diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1136,36 +1136,37 @@ nsubkwargs = 0 nkw = 0 nseen = 0 # the number of positional arguments on the stack - for elt in args: - if isinstance(elt, ast.Starred): - # A star-arg. If we've seen positional arguments, - # pack the positional arguments into a tuple. - if nseen != 0: - ops.BUILD_TUPLE(nseen) - nseen = 0 + if args is not None: + for elt in args: + if isinstance(elt, ast.Starred): + # A star-arg. If we've seen positional arguments, + # pack the positional arguments into a tuple. + if nseen != 0: + self.emit_op_arg(ops.BUILD_TUPLE, nseen) + nseen = 0 + nsubargs += 1 + elt.value.walkabout(self) #self.visit(elt.value) # check orig, elt->v.Starred.value nsubargs += 1 - elt.walkabout(self) #self.visit(elt.value) # probably wrong, elt->v.Starred.value + elif nsubargs != 0: + # We've seen star-args already, so we + # count towards items-to-pack-into-tuple. + elt.walkabout(self) + nseen += 1 + else: + # Positional arguments before star-arguments + # are left on the stack. + elt.walkabout(self) + n += 1 + if nseen != 0: + # Pack up any trailing positional arguments. + self.emit_op_arg(ops.BUILD_TUPLE, nseen) nsubargs += 1 - elif nsubargs != 0: - # We've seen star-args already, so we - # count towards items-to-pack-into-tuple. - elt.walkabout(self) - nseen += 1 - else: - # Positional arguments before star-arguments - # are left on the stack. - elt.walkabout(self) - n += 1 - if nseen != 0: - # Pack up any trailing positional arguments. - ops.BUILD_TUPLE(nseen) - nsubargs += 1 - if nsubargs != 0: - call_type |= 1 - if nsubargs > 1: - # If we ended up with more than one stararg, we need - # to concatenate them into a single sequence. - ops.BUILD_LIST_UNPACK(nsubargs) + if nsubargs != 0: + call_type |= 1 + if nsubargs > 1: + # If we ended up with more than one stararg, we need + # to concatenate them into a single sequence. + self.emit_op_arg(ops.BUILD_LIST_UNPACK, nsubargs) # Repeat procedure for keyword args nseen = 0 # the number of keyword arguments on the stack following @@ -1174,30 +1175,30 @@ if kw.arg is None: # A keyword argument unpacking. if nseen != 0: - ops.BUILD_MAP(nseen) + self.emit_op_arg(ops.BUILD_MAP, nseen) nseen = 0 nsubkwargs += 1 - self.visit_sequence(kw.value) # probably wrong, elt->v.Starred.value + kw.value.walkabout(self) #self.visit_sequence(kw.value) # probably wrong, elt->v.Starred.value nsubkwargs += 1 elif nsubkwargs != 0: # A keyword argument and we already have a dict. - ops.LOAD_CONST(kw.arg, consts) - self.visit_sequence(kw.value) + self.load_const(kw.arg) + kw.value.walkabout(self) nseen += 1 else: # keyword argument - self.visit_sequence(kw) + kw.walkabout(self) nkw += 1 if nseen != 0: # Pack up any trailing keyword arguments. - ops.BUILD_MAP(nseen) + self.emit_op_arg(ops.BUILD_MAP,nseen) nsubkwargs += 1 if nsubargs != 0: call_type |= 2 if nsubkwargs > 1: # Pack it all up function_pos = n + (code & 1) + nkw + 1 - ops.BUILD_MAP_UNPACK_WITH_CALL(nsubkwargs | (function_pos << 8)) + self.emit_op_arg(ops.BUILD_MAP_UNPACK_WITH_CALL, (nsubkwargs | (function_pos << 8))) assert n < 1<<8 assert nkw < 1<<24 @@ -1213,7 +1214,6 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) - #TODO emip_op_arg on each call def visit_Call(self, call): self.update_position(call.lineno) From pypy.commits at gmail.com Mon Jun 6 12:22:23 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 09:22:23 -0700 (PDT) Subject: [pypy-commit] pypy default: fix for 9f49da4855da Message-ID: <5755a33f.831dc20a.9d642.fffffa48@mx.google.com> Author: Matti Picus Branch: Changeset: r84970:4b396d7cbc4c Date: 2016-06-06 19:16 +0300 http://bitbucket.org/pypy/pypy/changeset/4b396d7cbc4c/ Log: fix for 9f49da4855da diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,7 @@ import py import pytest + at pytest.mark.trylast def pytest_configure(config): from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) From pypy.commits at gmail.com Mon Jun 6 12:22:25 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 09:22:25 -0700 (PDT) Subject: [pypy-commit] pypy default: fix signature Message-ID: <5755a341.43921c0a.2b3ee.31f8@mx.google.com> Author: Matti Picus Branch: Changeset: r84971:f4d726d1a010 Date: 2016-06-06 19:20 +0300 http://bitbucket.org/pypy/pypy/changeset/f4d726d1a010/ Log: fix signature diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, - PyVarObject, Py_buffer, + PyVarObject, Py_buffer, size_t, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( @@ -14,14 +14,14 @@ import pypy.module.__builtin__.operation as operation - at cpython_api([Py_ssize_t], rffi.VOIDP) + at cpython_api([size_t], rffi.VOIDP) def PyObject_Malloc(space, size): # returns non-zero-initialized memory, like CPython return lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', add_memory_pressure=True) - at cpython_api([rffi.VOIDP, Py_ssize_t], rffi.VOIDP) + at cpython_api([rffi.VOIDP, size_t], rffi.VOIDP) def PyObject_Realloc(space, ptr, size): if not lltype.cast_ptr_to_int(ptr): return lltype.malloc(rffi.VOIDP.TO, size, From pypy.commits at gmail.com Mon Jun 6 12:22:26 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 09:22:26 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: merge default into release Message-ID: <5755a342.04251c0a.b4768.ffff8e32@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84972:e5257f0e43e9 Date: 2016-06-06 19:21 +0300 http://bitbucket.org/pypy/pypy/changeset/e5257f0e43e9/ Log: merge default into release diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst --- a/pypy/doc/release-pypy2.7-v5.3.0.rst +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -9,9 +9,11 @@ for lxml, we now pass most (more than 90%) of the upstream numpy test suite, and much of SciPy is supported as well. -We also improved the speed of ... and ... +We updated cffi_ to version 1.7 (small changes, documented here_). -We updated cffi_ to ... +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html +.. _cffi: https://cffi.readthedocs.org +.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html You can download the PyPy2.7 v5.3 release here: @@ -30,10 +32,6 @@ .. _`RPython`: https://rpython.readthedocs.org .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html -.. _`numpy`: https://bitbucket.org/pypy/numpy -.. _cffi: https://cffi.readthedocs.org -.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html -.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html What is PyPy? ============= @@ -42,8 +40,8 @@ CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -We also welcome developers of other -`dynamic languages`_ to see what RPython can do for them. +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. This release supports: @@ -107,12 +105,10 @@ * Revive traceviewer, a tool to use pygame to view traces - * Update to cffi/847bbc0297f8 which improves help() on cffi objects - * Bug Fixes - * Fix issue #2277: only special-case two exact lists in zip(), not list - subclasses, because an overridden __iter__() should be called (probably) + * Fix issue #2277: only special-case two exact lists in zip(), not list + subclasses, because an overridden __iter__() should be called (probably) * Fix issue #2226: Another tweak in the incremental GC- this should ensure that progress in the major GC occurs quickly enough in all cases. @@ -145,7 +141,7 @@ * Implement ufunc.outer on numpypy - * Move PyPy-specific numpy headers to a subdirectory (also changed pypy/numpy + * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_ accordingly) * Performance improvements: @@ -185,9 +181,9 @@ * Compile c snippets with -Werror, and fix warnings it exposed -.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html .. _Numpy: https://bitbucket.org/pypy/numpy +.. _`the repo`: https://bitbucket.org/pypy/numpy Please update, and continue to help us make PyPy better. diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,5 +1,5 @@ +import pytest from rpython.tool import udir -from pypy.conftest import option from pypy.interpreter.gateway import interp2app def check_no_w_locals(space, w_frame): @@ -11,7 +11,7 @@ space = cls.space cls.w_udir = cls.space.wrap(str(udir.udir)) cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) - if not option.runappdirect: + if not pytest.config.option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): return f() @@ -48,10 +48,10 @@ return f.f_code assert g() is g.func_code - def test_f_trace_del(self): + def test_f_trace_del(self): import sys - f = sys._getframe() - del f.f_trace + f = sys._getframe() + del f.f_trace assert f.f_trace is None def test_f_lineno(self): @@ -116,7 +116,7 @@ def f(): assert sys._getframe().f_code.co_name == g() def g(): - return sys._getframe().f_back.f_code.co_name + return sys._getframe().f_back.f_code.co_name f() def test_f_back_virtualref(self): @@ -233,7 +233,7 @@ def test_trace_exc(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -298,7 +298,7 @@ def test_trace_return_exc(self): import sys l = [] - def trace(a,b,c): + def trace(a,b,c): if b in ('exception', 'return'): l.append((b, c)) return trace @@ -444,7 +444,7 @@ def test_dont_trace_on_reraise(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -466,7 +466,7 @@ def test_dont_trace_on_raise_with_tb(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -1,9 +1,8 @@ import py -from pypy import conftest from pypy.interpreter import gateway from rpython.rlib.jit import non_virtual_ref, vref_None -class AppTestSlow: +class AppTestSlow: spaceconfig = dict(usemodules=['itertools']) def setup_class(cls): @@ -64,7 +63,7 @@ space.setitem(space.builtin.w_dict, space.wrap('read_exc_type'), space.wrap(read_exc_type_gw)) - + def _detach_helpers(space): space.delitem(space.builtin.w_dict, space.wrap('hide_top_frame')) @@ -92,7 +91,7 @@ pckl = pickle.dumps(code) result = pickle.loads(pckl) assert code == result - + def test_pickle_global_func(self): import new mod = new.module('mod') @@ -109,7 +108,7 @@ assert func is result finally: del sys.modules['mod'] - + def test_pickle_not_imported_module(self): import new mod = new.module('mod') @@ -119,13 +118,13 @@ result = pickle.loads(pckl) assert mod.__name__ == result.__name__ assert mod.__dict__ == result.__dict__ - + def test_pickle_builtin_func(self): import pickle pckl = pickle.dumps(map) result = pickle.loads(pckl) assert map is result - + def test_pickle_non_top_reachable_func(self): def func(): return 42 @@ -142,7 +141,7 @@ assert func.func_dict == result.func_dict assert func.func_doc == result.func_doc assert func.func_globals == result.func_globals - + def test_pickle_cell(self): def g(): x = [42] @@ -171,7 +170,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) assert type(f1) is type(f2) @@ -223,7 +222,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) def test_frame_setstate_crash(self): @@ -257,21 +256,21 @@ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_moduledict(self): import pickle moddict = pickle.__dict__ pckl = pickle.dumps(moddict) result = pickle.loads(pckl) assert moddict is result - + def test_pickle_bltins_module(self): import pickle mod = __builtins__ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_buffer(self): skip("Can't pickle buffer objects on top of CPython either. " "Do we really need it?") @@ -280,14 +279,14 @@ pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_complex(self): import pickle a = complex(1.23,4.567) pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_method(self): class myclass(object): def f(self): @@ -308,7 +307,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_staticmethod(self): class myclass(object): def f(): @@ -319,7 +318,7 @@ pckl = pickle.dumps(method) result = pickle.loads(pckl) assert method() == result() - + def test_pickle_classmethod(self): class myclass(object): def f(cls): @@ -337,7 +336,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_sequenceiter(self): ''' In PyPy there is no distinction here between listiterator and diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -420,6 +420,14 @@ w_result = ctype.ctitem.unpack_ptr(ctype, ptr, length) return w_result + def dir(self, space): + from pypy.module._cffi_backend.ctypeptr import W_CTypePointer + ct = self.ctype + if isinstance(ct, W_CTypePointer): + ct = ct.ctitem + lst = ct.cdata_dir() + return space.newlist([space.wrap(s) for s in lst]) + class W_CDataMem(W_CData): """This is used only by the results of cffi.cast('int', x) @@ -602,5 +610,6 @@ __call__ = interp2app(W_CData.call), __iter__ = interp2app(W_CData.iter), __weakref__ = make_weakref_descr(W_CData), + __dir__ = interp2app(W_CData.dir), ) W_CData.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -256,6 +256,9 @@ def fget_elements(self, space): return self._fget('e') def fget_relements(self, space):return self._fget('R') + def cdata_dir(self): + return [] + W_CType.typedef = TypeDef( '_cffi_backend.CTypeDescr', diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -171,6 +171,12 @@ pass return W_CType.getcfield(self, attr) + def cdata_dir(self): + if self.size < 0: + return [] + self.force_lazy_struct() + return self._fields_dict.keys() + class W_CTypeStruct(W_CTypeStructOrUnion): kind = "struct" diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -77,8 +77,8 @@ assert repr(p) == "" def check_dir(p, expected): - got = set(name for name in dir(p) if not name.startswith('_')) - assert got == set(expected) + got = [name for name in dir(p) if not name.startswith('_')] + assert got == sorted(expected) def test_inspect_primitive_type(): p = new_primitive_type("signed char") @@ -3608,3 +3608,23 @@ # py.test.raises(ValueError, unpack, p0, -1) py.test.raises(ValueError, unpack, p, -1) + +def test_cdata_dir(): + BInt = new_primitive_type("int") + p = cast(BInt, 42) + check_dir(p, []) + p = newp(new_array_type(new_pointer_type(BInt), None), 5) + check_dir(p, []) + BStruct = new_struct_type("foo") + p = cast(new_pointer_type(BStruct), 0) + check_dir(p, []) # opaque + complete_struct_or_union(BStruct, [('a2', BInt, -1), + ('a1', BInt, -1)]) + check_dir(p, ['a1', 'a2']) # always sorted + p = newp(new_pointer_type(BStruct), None) + check_dir(p, ['a1', 'a2']) + check_dir(p[0], ['a1', 'a2']) + pp = newp(new_pointer_type(new_pointer_type(BStruct)), p) + check_dir(pp, []) + check_dir(pp[0], ['a1', 'a2']) + check_dir(pp[0][0], ['a1', 'a2']) diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -89,8 +89,7 @@ class AppTestWrapper: def setup_class(cls): "Run test_various_depths() when we are run with 'pypy py.test -A'." - from pypy.conftest import option - if not option.runappdirect: + if not py.test.config.option.runappdirect: py.test.skip("meant only for -A run") def _setup(): diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -516,12 +516,11 @@ assert s == 'bar\n' def test_flush_at_exit(): - from pypy import conftest from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) + config = make_config(py.test.config.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): f = open(tmpfile, 'w') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -222,14 +222,10 @@ assert not closed[0] # flush() called before file closed os.close(fd) -def test_flush_at_exit(): - from pypy import conftest - from pypy.tool.option import make_config, make_objspace +def test_flush_at_exit(space): from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) - space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): import io f = io.open(tmpfile, 'w', encoding='ascii') @@ -241,12 +237,7 @@ assert tmpfile.read() == '42' -def test_flush_at_exit_IOError_and_ValueError(): - from pypy import conftest - from pypy.tool.option import make_config, make_objspace - - config = make_config(conftest.option) - space = make_objspace(config) +def test_flush_at_exit_IOError_and_ValueError(space): space.appexec([], """(): import io class MyStream(io.IOBase): diff --git a/pypy/module/_rawffi/test/test_tracker.py b/pypy/module/_rawffi/test/test_tracker.py --- a/pypy/module/_rawffi/test/test_tracker.py +++ b/pypy/module/_rawffi/test/test_tracker.py @@ -1,5 +1,5 @@ import py -from pypy.conftest import option +from pytest import option from pypy.module._rawffi.tracker import Tracker @@ -44,4 +44,3 @@ def teardown_class(cls): Tracker.DO_TRACING = False - diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, - PyVarObject, Py_buffer, + PyVarObject, Py_buffer, size_t, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( @@ -14,14 +14,14 @@ import pypy.module.__builtin__.operation as operation - at cpython_api([Py_ssize_t], rffi.VOIDP) + at cpython_api([size_t], rffi.VOIDP) def PyObject_Malloc(space, size): # returns non-zero-initialized memory, like CPython return lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', add_memory_pressure=True) - at cpython_api([rffi.VOIDP, Py_ssize_t], rffi.VOIDP) + at cpython_api([rffi.VOIDP, size_t], rffi.VOIDP) def PyObject_Realloc(space, ptr, size): if not lltype.cast_ptr_to_int(ptr): return lltype.malloc(rffi.VOIDP.TO, size, diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,7 @@ import py import pytest + at pytest.mark.trylast def pytest_configure(config): from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) diff --git a/pypy/module/gc/test/test_referents.py b/pypy/module/gc/test/test_referents.py --- a/pypy/module/gc/test/test_referents.py +++ b/pypy/module/gc/test/test_referents.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from pytest import option class AppTestReferents(object): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -13,7 +13,7 @@ from pypy.module.imp import importing -from pypy import conftest +from pytest import config def setuppkg(pkgname, **entries): p = udir.join('impsubdir') @@ -106,7 +106,7 @@ # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") - if conftest.option.runappdirect: + if config.option.runappdirect: import marshal, stat, struct, os, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) @@ -143,7 +143,7 @@ def _setup(space): dn = setup_directory_structure(space) return space.appexec([space.wrap(dn)], """ - (dn): + (dn): import sys path = list(sys.path) sys.path.insert(0, dn) @@ -168,7 +168,7 @@ } def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -1017,7 +1017,7 @@ cpathname = udir.join('test.pyc') assert not cpathname.check() - + def test_load_source_module_importerror(self): # the .pyc file is created before executing the module space = self.space @@ -1126,11 +1126,11 @@ stream.close() -def test_PYTHONPATH_takes_precedence(space): +def test_PYTHONPATH_takes_precedence(space): if sys.platform == "win32": py.test.skip("unresolved issues with win32 shell quoting rules") - from pypy.interpreter.test.test_zpy import pypypath - extrapath = udir.ensure("pythonpath", dir=1) + from pypy.interpreter.test.test_zpy import pypypath + extrapath = udir.ensure("pythonpath", dir=1) extrapath.join("sched.py").write("print 42\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) @@ -1494,8 +1494,6 @@ spaceconfig = dict(usemodules=['thread', 'time']) def setup_class(cls): - #if not conftest.option.runappdirect: - # py.test.skip("meant as an -A test") tmpfile = udir.join('test_multithreaded_imp.py') tmpfile.write('''if 1: x = 666 diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy import constants as NPY @@ -7,7 +7,7 @@ @classmethod def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: import sys if '__pypy__' not in sys.builtin_module_names: import numpy diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -2,7 +2,6 @@ import sys -from pypy.conftest import option from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest @@ -23,13 +22,13 @@ # special values testing if isnan(a): if isnan(b): - return True,'' + return True, '' raise AssertionError(msg + '%r should be nan' % (b,)) if isinf(a): if a == b: - return True,'' - raise AssertionError(msg + 'finite result where infinity expected: '+ \ + return True, '' + raise AssertionError(msg + 'finite result where infinity expected: ' 'expected %r, got %r' % (a, b)) # if both a and b are zero, check whether they have the same sign @@ -39,7 +38,7 @@ if not a and not b: # only check it if we are running on top of CPython >= 2.6 if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b): - raise AssertionError( msg + \ + raise AssertionError(msg + 'zero has wrong sign: expected %r, got %r' % (a, b)) # if a-b overflows, or b is infinite, return False. Again, in @@ -96,7 +95,6 @@ cls.w_testcases128 = cls.space.wrap(list(parse_testfile(fname128))) cls.w_testcases64 = cls.space.wrap(list(parse_testfile(fname64))) - cls.w_runAppDirect = cls.space.wrap(option.runappdirect) cls.w_isWindows = cls.space.wrap(os.name == 'nt') if cls.runappdirect: @@ -495,8 +493,8 @@ c = array([1.e+110, 1.e-110], dtype=complex128) d = floor_divide(c**2, c) assert (d == [1.e+110, 0]).all() - - + + def test_basic(self): import sys diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1,11 +1,11 @@ -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.interpreter.gateway import interp2app class BaseAppTestDtypes(BaseNumpyAppTest): def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if option.runappdirect: + if config.option.runappdirect: import platform bits, linkage = platform.architecture() ptr_size = int(bits[:-3]) // 8 @@ -372,8 +372,8 @@ a = np.array(data, dtype=b) x = pickle.loads(pickle.dumps(a)) assert (x == a).all() - assert x.dtype == a.dtype - + assert x.dtype == a.dtype + def test_index(self): import numpy as np for dtype in [np.int8, np.int16, np.int32, np.int64]: @@ -1088,7 +1088,7 @@ spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if option.runappdirect: + if config.option.runappdirect: cls.w_test_for_core_internal = cls.space.wrap(True) else: cls.w_test_for_core_internal = cls.space.wrap(False) @@ -1459,7 +1459,7 @@ "'offsets':[0,76800], " "'itemsize':80000, " "'aligned':True}") - + assert dt == np.dtype(eval(str(dt))) dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'], @@ -1515,7 +1515,7 @@ else: assert stor2[1] == '\x01' assert stor2[0] == '\x00' - if option.runappdirect: + if config.option.runappdirect: cls.w_check_non_native = lambda *args : None else: cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2,7 +2,7 @@ import py import sys -from pypy.conftest import option +from pytest import config from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk from pypy.module.micronumpy.ndarray import W_NDimArray @@ -1878,7 +1878,7 @@ assert map(isnan, e) == [False, False, False, True, False] assert map(isinf, e) == [False, False, True, False, False] assert e.argmax() == 3 - # numpy preserves value for uint16 -> cast_as_float16 -> + # numpy preserves value for uint16 -> cast_as_float16 -> # convert_to_float64 -> convert_to_float16 -> uint16 # even for float16 various float16 nans all_f16 = arange(0xfe00, 0xffff, dtype='uint16') @@ -2608,7 +2608,7 @@ a = np.arange(6).reshape(2,3) i = np.dtype('int32').type(0) assert (a[0] == a[i]).all() - + def test_ellipsis_indexing(self): import numpy as np @@ -3850,7 +3850,7 @@ class AppTestRepr(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: py.test.skip("Can't be run directly.") BaseNumpyAppTest.setup_class.im_func(cls) cache = get_appbridge_cache(cls.space) @@ -3867,7 +3867,7 @@ assert repr(array(1.5).real) == "array(1.5)" def teardown_class(cls): - if option.runappdirect: + if config.option.runappdirect: return cache = get_appbridge_cache(cls.space) cache.w_array_repr = cls.old_array_repr @@ -4343,7 +4343,7 @@ class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if config.option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -1,14 +1,9 @@ from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pypy.conftest import option class AppTestObjectDtypes(BaseNumpyAppTest): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) - def setup_class(cls): - BaseNumpyAppTest.setup_class.im_func(cls) - cls.w_runappdirect = cls.space.wrap(option.runappdirect) - def test_scalar_from_object(self): from numpy import array import sys @@ -200,7 +195,7 @@ from numpy import arange, dtype from cPickle import loads, dumps import sys - + a = arange(15).astype(object) if '__pypy__' in sys.builtin_module_names: raises(NotImplementedError, dumps, a) @@ -211,4 +206,4 @@ a = arange(15).astype(object).reshape((3, 5)) b = loads(dumps(a)) assert (a == b).all() - + diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py --- a/pypy/module/micronumpy/test/test_support_app.py +++ b/pypy/module/micronumpy/test/test_support_app.py @@ -3,11 +3,12 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pypy.conftest import option +from pytest import config class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): - if option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if (config.option.runappdirect and + '__pypy__' not in sys.builtin_module_names): py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/test_lib_pypy/support.py b/pypy/module/test_lib_pypy/support.py --- a/pypy/module/test_lib_pypy/support.py +++ b/pypy/module/test_lib_pypy/support.py @@ -1,6 +1,6 @@ import py -from pypy.conftest import option +from pytest import config from pypy.interpreter.error import OperationError def import_lib_pypy(space, name, skipmsg=None): @@ -9,7 +9,7 @@ Raises a pytest Skip on ImportError if a skip message was specified. """ - if option.runappdirect: + if config.option.runappdirect: try: mod = __import__('lib_pypy.' + name) except ImportError as e: diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,9 +1,9 @@ -from pypy import conftest +from pytest import config class AppTestBytesArray: def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) def test_basics(self): b = bytearray() diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -1,5 +1,4 @@ -from __future__ import with_statement -from pypy.conftest import option +from pytest import config class AppTestObject: @@ -7,13 +6,13 @@ from pypy.interpreter import gateway import sys - cpython_behavior = (not option.runappdirect + cpython_behavior = (not config.option.runappdirect or not hasattr(sys, 'pypy_translation_info')) space = cls.space cls.w_cpython_behavior = space.wrap(cpython_behavior) cls.w_cpython_version = space.wrap(tuple(sys.version_info)) - cls.w_appdirect = space.wrap(option.runappdirect) + cls.w_appdirect = space.wrap(config.option.runappdirect) cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info')) def w_unwrap_wrap_unicode(space, w_obj): diff --git a/pypy/objspace/std/test/test_proxy_usercreated.py b/pypy/objspace/std/test/test_proxy_usercreated.py --- a/pypy/objspace/std/test/test_proxy_usercreated.py +++ b/pypy/objspace/std/test/test_proxy_usercreated.py @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app from pypy.objspace.std.transparent import register_proxyable -from pypy.conftest import option +from pytest import config class W_Wrapped(W_Root): @@ -25,7 +25,7 @@ class AppTestProxyNewtype(AppProxy): def setup_class(cls): - if option.runappdirect: + if config.option.runappdirect: py.test.skip("Impossible to run on appdirect") AppProxy.setup_class.im_func(cls) cls.w_wrapped = cls.space.wrap(W_Wrapped()) diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py --- a/pypy/objspace/test/test_binop_overriding.py +++ b/pypy/objspace/test/test_binop_overriding.py @@ -1,5 +1,5 @@ # test about the binop operation rule, see issue 412 -from pypy.conftest import option +from pytest import config class AppTestBinopCombinations: @@ -73,7 +73,7 @@ if C is not object: setattr(C, name, f) override_in_hier(n-1) - if C is not object: + if C is not object: delattr(C, name) override_in_hier() @@ -83,7 +83,7 @@ return Base, do_test """) cls.w_helpers = w_helpers - cls.w_appdirect = cls.space.wrap(option.runappdirect) + cls.w_appdirect = cls.space.wrap(config.option.runappdirect) def test_overriding_base_binop_explict(self): class MulBase(object): @@ -105,7 +105,7 @@ if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -116,7 +116,7 @@ assert not fail def test_binop_combinations_sub(self): - Base, do_test = self.helpers + Base, do_test = self.helpers class X(Base): pass class Y(X): @@ -124,13 +124,13 @@ fail = do_test(X, Y, 'sub', lambda x,y: x-y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_pow(self): if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -138,13 +138,13 @@ fail = do_test(X, Y, 'pow', lambda x,y: x**y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_more_exhaustive(self): if not self.appdirect: skip("very slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -184,7 +184,6 @@ source = str(source).strip() except py.error.ENOENT: source = None - from pypy import conftest if source and py.test.config._assertstate.mode != "off": msg = interpret(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -3,14 +3,13 @@ from rpython.config.config import ConflictConfigError from pypy.tool.option import make_config, make_objspace from pypy.tool.pytest import appsupport -from pypy.conftest import option _SPACECACHE={} def gettestobjspace(**kwds): """ helper for instantiating and caching space's for testing. """ try: - config = make_config(option,**kwds) + config = make_config(py.test.config.option,**kwds) except ConflictConfigError as e: # this exception is typically only raised if a module is not available. # in this case the test should be skipped @@ -19,7 +18,7 @@ try: return _SPACECACHE[key] except KeyError: - if getattr(option, 'runappdirect', None): + if getattr(py.test.config.option, 'runappdirect', None): return TinyObjSpace(**kwds) space = maketestobjspace(config) _SPACECACHE[key] = space @@ -27,7 +26,7 @@ def maketestobjspace(config=None): if config is None: - config = make_config(option) + config = make_config(py.test.config.option) if config.objspace.usemodules.thread: config.translation.thread = True space = make_objspace(config) From pypy.commits at gmail.com Mon Jun 6 12:39:14 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 09:39:14 -0700 (PDT) Subject: [pypy-commit] pypy default: Don't use leakfinder when running with -A Message-ID: <5755a732.e7c9c20a.47bae.6f59@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84973:bd58f903f506 Date: 2016-06-06 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/bd58f903f506/ Log: Don't use leakfinder when running with -A diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -23,11 +23,10 @@ def pytest_report_header(): return "pytest-%s from %s" % (pytest.__version__, pytest.__file__) -def pytest_addhooks(pluginmanager): - from rpython.conftest import LeakFinder - pluginmanager.register(LeakFinder()) - def pytest_configure(config): + if not config.option.runappdirect: + from rpython.conftest import LeakFinder + config.pluginmanager.register(LeakFinder()) global option option = config.option From pypy.commits at gmail.com Mon Jun 6 13:01:27 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 10:01:27 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Handle partial write() Message-ID: <5755ac67.4fa51c0a.d046d.50ef@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r84974:58ec0aa13c9b Date: 2016-06-06 19:02 +0200 http://bitbucket.org/pypy/pypy/changeset/58ec0aa13c9b/ Log: Handle partial write() diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -62,14 +62,28 @@ { /* write the current buffer content to the OS */ - ssize_t size = rpy_revdb.buf_p - rpy_rev_buffer; + ssize_t wsize, size = rpy_revdb.buf_p - rpy_rev_buffer; + char *p; rpy_revdb.buf_p = rpy_rev_buffer; - if (size > 0 && rpy_rev_fileno >= 0) { - if (write(rpy_rev_fileno, rpy_rev_buffer, size) != size) { + if (size == 0 || rpy_rev_fileno < 0) + return; + + p = rpy_rev_buffer; + retry: + wsize = write(rpy_rev_fileno, p, size); + if (wsize >= size) + return; + if (wsize <= 0) { + if (wsize == 0) + fprintf(stderr, "Writing to PYPYREVDB file: " + "unexpected non-blocking mode\n"); + else fprintf(stderr, "Fatal error: writing to PYPYREVDB file: %m\n"); - abort(); - } + abort(); } + p += wsize; + size -= wsize; + goto retry; } From pypy.commits at gmail.com Mon Jun 6 13:57:38 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 10:57:38 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: Backed out changeset: 9f49da4855da Message-ID: <5755b992.665ec20a.79856.ffff956a@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84975:ef4abbf7419d Date: 2016-06-06 20:55 +0300 http://bitbucket.org/pypy/pypy/changeset/ef4abbf7419d/ Log: Backed out changeset: 9f49da4855da diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,5 +1,5 @@ -import pytest from rpython.tool import udir +from pypy.conftest import option from pypy.interpreter.gateway import interp2app def check_no_w_locals(space, w_frame): @@ -11,7 +11,7 @@ space = cls.space cls.w_udir = cls.space.wrap(str(udir.udir)) cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) - if not pytest.config.option.runappdirect: + if not option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): return f() @@ -48,10 +48,10 @@ return f.f_code assert g() is g.func_code - def test_f_trace_del(self): + def test_f_trace_del(self): import sys - f = sys._getframe() - del f.f_trace + f = sys._getframe() + del f.f_trace assert f.f_trace is None def test_f_lineno(self): @@ -116,7 +116,7 @@ def f(): assert sys._getframe().f_code.co_name == g() def g(): - return sys._getframe().f_back.f_code.co_name + return sys._getframe().f_back.f_code.co_name f() def test_f_back_virtualref(self): @@ -233,7 +233,7 @@ def test_trace_exc(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -298,7 +298,7 @@ def test_trace_return_exc(self): import sys l = [] - def trace(a,b,c): + def trace(a,b,c): if b in ('exception', 'return'): l.append((b, c)) return trace @@ -444,7 +444,7 @@ def test_dont_trace_on_reraise(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -466,7 +466,7 @@ def test_dont_trace_on_raise_with_tb(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -1,8 +1,9 @@ import py +from pypy import conftest from pypy.interpreter import gateway from rpython.rlib.jit import non_virtual_ref, vref_None -class AppTestSlow: +class AppTestSlow: spaceconfig = dict(usemodules=['itertools']) def setup_class(cls): @@ -63,7 +64,7 @@ space.setitem(space.builtin.w_dict, space.wrap('read_exc_type'), space.wrap(read_exc_type_gw)) - + def _detach_helpers(space): space.delitem(space.builtin.w_dict, space.wrap('hide_top_frame')) @@ -91,7 +92,7 @@ pckl = pickle.dumps(code) result = pickle.loads(pckl) assert code == result - + def test_pickle_global_func(self): import new mod = new.module('mod') @@ -108,7 +109,7 @@ assert func is result finally: del sys.modules['mod'] - + def test_pickle_not_imported_module(self): import new mod = new.module('mod') @@ -118,13 +119,13 @@ result = pickle.loads(pckl) assert mod.__name__ == result.__name__ assert mod.__dict__ == result.__dict__ - + def test_pickle_builtin_func(self): import pickle pckl = pickle.dumps(map) result = pickle.loads(pckl) assert map is result - + def test_pickle_non_top_reachable_func(self): def func(): return 42 @@ -141,7 +142,7 @@ assert func.func_dict == result.func_dict assert func.func_doc == result.func_doc assert func.func_globals == result.func_globals - + def test_pickle_cell(self): def g(): x = [42] @@ -170,7 +171,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) assert type(f1) is type(f2) @@ -222,7 +223,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) def test_frame_setstate_crash(self): @@ -256,21 +257,21 @@ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_moduledict(self): import pickle moddict = pickle.__dict__ pckl = pickle.dumps(moddict) result = pickle.loads(pckl) assert moddict is result - + def test_pickle_bltins_module(self): import pickle mod = __builtins__ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_buffer(self): skip("Can't pickle buffer objects on top of CPython either. " "Do we really need it?") @@ -279,14 +280,14 @@ pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_complex(self): import pickle a = complex(1.23,4.567) pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_method(self): class myclass(object): def f(self): @@ -307,7 +308,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_staticmethod(self): class myclass(object): def f(): @@ -318,7 +319,7 @@ pckl = pickle.dumps(method) result = pickle.loads(pckl) assert method() == result() - + def test_pickle_classmethod(self): class myclass(object): def f(cls): @@ -336,7 +337,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_sequenceiter(self): ''' In PyPy there is no distinction here between listiterator and diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -89,7 +89,8 @@ class AppTestWrapper: def setup_class(cls): "Run test_various_depths() when we are run with 'pypy py.test -A'." - if not py.test.config.option.runappdirect: + from pypy.conftest import option + if not option.runappdirect: py.test.skip("meant only for -A run") def _setup(): diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -516,11 +516,12 @@ assert s == 'bar\n' def test_flush_at_exit(): + from pypy import conftest from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(py.test.config.option) + config = make_config(conftest.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): f = open(tmpfile, 'w') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -222,10 +222,14 @@ assert not closed[0] # flush() called before file closed os.close(fd) -def test_flush_at_exit(space): +def test_flush_at_exit(): + from pypy import conftest + from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') + config = make_config(conftest.option) + space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): import io f = io.open(tmpfile, 'w', encoding='ascii') @@ -237,7 +241,12 @@ assert tmpfile.read() == '42' -def test_flush_at_exit_IOError_and_ValueError(space): +def test_flush_at_exit_IOError_and_ValueError(): + from pypy import conftest + from pypy.tool.option import make_config, make_objspace + + config = make_config(conftest.option) + space = make_objspace(config) space.appexec([], """(): import io class MyStream(io.IOBase): diff --git a/pypy/module/_rawffi/test/test_tracker.py b/pypy/module/_rawffi/test/test_tracker.py --- a/pypy/module/_rawffi/test/test_tracker.py +++ b/pypy/module/_rawffi/test/test_tracker.py @@ -1,5 +1,5 @@ import py -from pytest import option +from pypy.conftest import option from pypy.module._rawffi.tracker import Tracker @@ -44,3 +44,4 @@ def teardown_class(cls): Tracker.DO_TRACING = False + diff --git a/pypy/module/gc/test/test_referents.py b/pypy/module/gc/test/test_referents.py --- a/pypy/module/gc/test/test_referents.py +++ b/pypy/module/gc/test/test_referents.py @@ -1,4 +1,4 @@ -from pytest import option +from pypy.conftest import option class AppTestReferents(object): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -13,7 +13,7 @@ from pypy.module.imp import importing -from pytest import config +from pypy import conftest def setuppkg(pkgname, **entries): p = udir.join('impsubdir') @@ -106,7 +106,7 @@ # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") - if config.option.runappdirect: + if conftest.option.runappdirect: import marshal, stat, struct, os, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) @@ -143,7 +143,7 @@ def _setup(space): dn = setup_directory_structure(space) return space.appexec([space.wrap(dn)], """ - (dn): + (dn): import sys path = list(sys.path) sys.path.insert(0, dn) @@ -168,7 +168,7 @@ } def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -1017,7 +1017,7 @@ cpathname = udir.join('test.pyc') assert not cpathname.check() - + def test_load_source_module_importerror(self): # the .pyc file is created before executing the module space = self.space @@ -1126,11 +1126,11 @@ stream.close() -def test_PYTHONPATH_takes_precedence(space): +def test_PYTHONPATH_takes_precedence(space): if sys.platform == "win32": py.test.skip("unresolved issues with win32 shell quoting rules") - from pypy.interpreter.test.test_zpy import pypypath - extrapath = udir.ensure("pythonpath", dir=1) + from pypy.interpreter.test.test_zpy import pypypath + extrapath = udir.ensure("pythonpath", dir=1) extrapath.join("sched.py").write("print 42\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) @@ -1494,6 +1494,8 @@ spaceconfig = dict(usemodules=['thread', 'time']) def setup_class(cls): + #if not conftest.option.runappdirect: + # py.test.skip("meant as an -A test") tmpfile = udir.join('test_multithreaded_imp.py') tmpfile.write('''if 1: x = 666 diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -1,4 +1,4 @@ -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy import constants as NPY @@ -7,7 +7,7 @@ @classmethod def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: import sys if '__pypy__' not in sys.builtin_module_names: import numpy diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -2,6 +2,7 @@ import sys +from pypy.conftest import option from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest @@ -22,13 +23,13 @@ # special values testing if isnan(a): if isnan(b): - return True, '' + return True,'' raise AssertionError(msg + '%r should be nan' % (b,)) if isinf(a): if a == b: - return True, '' - raise AssertionError(msg + 'finite result where infinity expected: ' + return True,'' + raise AssertionError(msg + 'finite result where infinity expected: '+ \ 'expected %r, got %r' % (a, b)) # if both a and b are zero, check whether they have the same sign @@ -38,7 +39,7 @@ if not a and not b: # only check it if we are running on top of CPython >= 2.6 if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b): - raise AssertionError(msg + + raise AssertionError( msg + \ 'zero has wrong sign: expected %r, got %r' % (a, b)) # if a-b overflows, or b is infinite, return False. Again, in @@ -95,6 +96,7 @@ cls.w_testcases128 = cls.space.wrap(list(parse_testfile(fname128))) cls.w_testcases64 = cls.space.wrap(list(parse_testfile(fname64))) + cls.w_runAppDirect = cls.space.wrap(option.runappdirect) cls.w_isWindows = cls.space.wrap(os.name == 'nt') if cls.runappdirect: @@ -493,8 +495,8 @@ c = array([1.e+110, 1.e-110], dtype=complex128) d = floor_divide(c**2, c) assert (d == [1.e+110, 0]).all() - - + + def test_basic(self): import sys diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1,11 +1,11 @@ -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.interpreter.gateway import interp2app class BaseAppTestDtypes(BaseNumpyAppTest): def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if config.option.runappdirect: + if option.runappdirect: import platform bits, linkage = platform.architecture() ptr_size = int(bits[:-3]) // 8 @@ -372,8 +372,8 @@ a = np.array(data, dtype=b) x = pickle.loads(pickle.dumps(a)) assert (x == a).all() - assert x.dtype == a.dtype - + assert x.dtype == a.dtype + def test_index(self): import numpy as np for dtype in [np.int8, np.int16, np.int32, np.int64]: @@ -1088,7 +1088,7 @@ spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if config.option.runappdirect: + if option.runappdirect: cls.w_test_for_core_internal = cls.space.wrap(True) else: cls.w_test_for_core_internal = cls.space.wrap(False) @@ -1459,7 +1459,7 @@ "'offsets':[0,76800], " "'itemsize':80000, " "'aligned':True}") - + assert dt == np.dtype(eval(str(dt))) dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'], @@ -1515,7 +1515,7 @@ else: assert stor2[1] == '\x01' assert stor2[0] == '\x00' - if config.option.runappdirect: + if option.runappdirect: cls.w_check_non_native = lambda *args : None else: cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2,7 +2,7 @@ import py import sys -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk from pypy.module.micronumpy.ndarray import W_NDimArray @@ -1878,7 +1878,7 @@ assert map(isnan, e) == [False, False, False, True, False] assert map(isinf, e) == [False, False, True, False, False] assert e.argmax() == 3 - # numpy preserves value for uint16 -> cast_as_float16 -> + # numpy preserves value for uint16 -> cast_as_float16 -> # convert_to_float64 -> convert_to_float16 -> uint16 # even for float16 various float16 nans all_f16 = arange(0xfe00, 0xffff, dtype='uint16') @@ -2608,7 +2608,7 @@ a = np.arange(6).reshape(2,3) i = np.dtype('int32').type(0) assert (a[0] == a[i]).all() - + def test_ellipsis_indexing(self): import numpy as np @@ -3850,7 +3850,7 @@ class AppTestRepr(BaseNumpyAppTest): def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: py.test.skip("Can't be run directly.") BaseNumpyAppTest.setup_class.im_func(cls) cache = get_appbridge_cache(cls.space) @@ -3867,7 +3867,7 @@ assert repr(array(1.5).real) == "array(1.5)" def teardown_class(cls): - if config.option.runappdirect: + if option.runappdirect: return cache = get_appbridge_cache(cls.space) cache.w_array_repr = cls.old_array_repr @@ -4343,7 +4343,7 @@ class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): - if config.option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -1,9 +1,14 @@ from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest +from pypy.conftest import option class AppTestObjectDtypes(BaseNumpyAppTest): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) + def test_scalar_from_object(self): from numpy import array import sys @@ -195,7 +200,7 @@ from numpy import arange, dtype from cPickle import loads, dumps import sys - + a = arange(15).astype(object) if '__pypy__' in sys.builtin_module_names: raises(NotImplementedError, dumps, a) @@ -206,4 +211,4 @@ a = arange(15).astype(object).reshape((3, 5)) b = loads(dumps(a)) assert (a == b).all() - + diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py --- a/pypy/module/micronumpy/test/test_support_app.py +++ b/pypy/module/micronumpy/test/test_support_app.py @@ -3,12 +3,11 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pytest import config +from pypy.conftest import option class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): - if (config.option.runappdirect and - '__pypy__' not in sys.builtin_module_names): + if option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/test_lib_pypy/support.py b/pypy/module/test_lib_pypy/support.py --- a/pypy/module/test_lib_pypy/support.py +++ b/pypy/module/test_lib_pypy/support.py @@ -1,6 +1,6 @@ import py -from pytest import config +from pypy.conftest import option from pypy.interpreter.error import OperationError def import_lib_pypy(space, name, skipmsg=None): @@ -9,7 +9,7 @@ Raises a pytest Skip on ImportError if a skip message was specified. """ - if config.option.runappdirect: + if option.runappdirect: try: mod = __import__('lib_pypy.' + name) except ImportError as e: diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,9 +1,9 @@ -from pytest import config +from pypy import conftest class AppTestBytesArray: def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) def test_basics(self): b = bytearray() diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -1,4 +1,5 @@ -from pytest import config +from __future__ import with_statement +from pypy.conftest import option class AppTestObject: @@ -6,13 +7,13 @@ from pypy.interpreter import gateway import sys - cpython_behavior = (not config.option.runappdirect + cpython_behavior = (not option.runappdirect or not hasattr(sys, 'pypy_translation_info')) space = cls.space cls.w_cpython_behavior = space.wrap(cpython_behavior) cls.w_cpython_version = space.wrap(tuple(sys.version_info)) - cls.w_appdirect = space.wrap(config.option.runappdirect) + cls.w_appdirect = space.wrap(option.runappdirect) cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info')) def w_unwrap_wrap_unicode(space, w_obj): diff --git a/pypy/objspace/std/test/test_proxy_usercreated.py b/pypy/objspace/std/test/test_proxy_usercreated.py --- a/pypy/objspace/std/test/test_proxy_usercreated.py +++ b/pypy/objspace/std/test/test_proxy_usercreated.py @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app from pypy.objspace.std.transparent import register_proxyable -from pytest import config +from pypy.conftest import option class W_Wrapped(W_Root): @@ -25,7 +25,7 @@ class AppTestProxyNewtype(AppProxy): def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: py.test.skip("Impossible to run on appdirect") AppProxy.setup_class.im_func(cls) cls.w_wrapped = cls.space.wrap(W_Wrapped()) diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py --- a/pypy/objspace/test/test_binop_overriding.py +++ b/pypy/objspace/test/test_binop_overriding.py @@ -1,5 +1,5 @@ # test about the binop operation rule, see issue 412 -from pytest import config +from pypy.conftest import option class AppTestBinopCombinations: @@ -73,7 +73,7 @@ if C is not object: setattr(C, name, f) override_in_hier(n-1) - if C is not object: + if C is not object: delattr(C, name) override_in_hier() @@ -83,7 +83,7 @@ return Base, do_test """) cls.w_helpers = w_helpers - cls.w_appdirect = cls.space.wrap(config.option.runappdirect) + cls.w_appdirect = cls.space.wrap(option.runappdirect) def test_overriding_base_binop_explict(self): class MulBase(object): @@ -105,7 +105,7 @@ if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -116,7 +116,7 @@ assert not fail def test_binop_combinations_sub(self): - Base, do_test = self.helpers + Base, do_test = self.helpers class X(Base): pass class Y(X): @@ -124,13 +124,13 @@ fail = do_test(X, Y, 'sub', lambda x,y: x-y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_pow(self): if not self.appdirect: skip("slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass class Y(X): @@ -138,13 +138,13 @@ fail = do_test(X, Y, 'pow', lambda x,y: x**y) #print len(fail) - assert not fail + assert not fail def test_binop_combinations_more_exhaustive(self): if not self.appdirect: skip("very slow test, should be run as appdirect test") Base, do_test = self.helpers - + class X(Base): pass diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -184,6 +184,7 @@ source = str(source).strip() except py.error.ENOENT: source = None + from pypy import conftest if source and py.test.config._assertstate.mode != "off": msg = interpret(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -3,13 +3,14 @@ from rpython.config.config import ConflictConfigError from pypy.tool.option import make_config, make_objspace from pypy.tool.pytest import appsupport +from pypy.conftest import option _SPACECACHE={} def gettestobjspace(**kwds): """ helper for instantiating and caching space's for testing. """ try: - config = make_config(py.test.config.option,**kwds) + config = make_config(option,**kwds) except ConflictConfigError as e: # this exception is typically only raised if a module is not available. # in this case the test should be skipped @@ -18,7 +19,7 @@ try: return _SPACECACHE[key] except KeyError: - if getattr(py.test.config.option, 'runappdirect', None): + if getattr(option, 'runappdirect', None): return TinyObjSpace(**kwds) space = maketestobjspace(config) _SPACECACHE[key] = space @@ -26,7 +27,7 @@ def maketestobjspace(config=None): if config is None: - config = make_config(py.test.config.option) + config = make_config(option) if config.objspace.usemodules.thread: config.translation.thread = True space = make_objspace(config) From pypy.commits at gmail.com Mon Jun 6 14:08:25 2016 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 06 Jun 2016 11:08:25 -0700 (PDT) Subject: [pypy-commit] pypy applevel-unroll-safe: experimental hack: implement @unroll_safe at app-level Message-ID: <5755bc19.d4d71c0a.5410a.59b3@mx.google.com> Author: Carl Friedrich Bolz Branch: applevel-unroll-safe Changeset: r84976:b2124aa81410 Date: 2016-06-06 17:17 +0200 http://bitbucket.org/pypy/pypy/changeset/b2124aa81410/ Log: experimental hack: implement @unroll_safe at app-level diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py --- a/lib-python/2.7/opcode.py +++ b/lib-python/2.7/opcode.py @@ -194,5 +194,6 @@ def_op('CALL_METHOD', 202) # #args not including 'self' def_op('BUILD_LIST_FROM_ARG', 203) jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements +jabs_op('JUMP_ABSOLUTE_UNROLL', 205) # "" del def_op, name_op, jrel_op, jabs_op diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -657,6 +657,7 @@ ops.JUMP_FORWARD: 0, ops.JUMP_ABSOLUTE: 0, + ops.JUMP_ABSOLUTE_UNROLL: 0, ops.JUMP_IF_TRUE_OR_POP: 0, ops.JUMP_IF_FALSE_OR_POP: 0, ops.POP_JUMP_IF_TRUE: -1, diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -198,6 +198,8 @@ return next_instr elif opcode == opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, ec) + elif opcode == opcodedesc.JUMP_ABSOLUTE_UNROLL.index: + return self.jump_absolute_unroll(oparg, ec) elif opcode == opcodedesc.BREAK_LOOP.index: next_instr = self.BREAK_LOOP(oparg, next_instr) elif opcode == opcodedesc.CONTINUE_LOOP.index: @@ -1055,6 +1057,10 @@ check_nonneg(jumpto) return jumpto + def jump_absolute_unroll(self, jumpto, ec): + check_nonneg(jumpto) + return jumpto + def JUMP_FORWARD(self, jumpby, next_instr): next_instr += jumpby return next_instr diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -59,6 +59,7 @@ class Module(MixedModule): appleveldefs = { + '_unroll_safe' : 'app_magic._unroll_safe', } interpleveldefs = { @@ -90,7 +91,7 @@ 'save_module_content_for_future_reload': 'interp_magic.save_module_content_for_future_reload', 'decode_long' : 'interp_magic.decode_long', - '_promote' : 'interp_magic._promote', + '_promote' : 'interp_magic._promote', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/app_magic.py b/pypy/module/__pypy__/app_magic.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/app_magic.py @@ -0,0 +1,42 @@ + + +def _unroll_safe(func): + """ Decorator to mark a function as unroll-safe, meaning the JIT will not + trace any of the loops in the function. Instead, the loops will be unrolled + fully into the caller. + + This function is experimental. + """ + # bit of a hack: replace JUMP_ABSOLUTE bytecode with JUMP_ABSOLUTE_UNROLL, + # which will not trigger jitting + from opcode import opname, HAVE_ARGUMENT, EXTENDED_ARG, opmap + from types import CodeType, FunctionType + code = list(func.func_code.co_code) + n = len(code) + i = 0 + replaced = False + while i < n: + orig_i = i + c = code[i] + op = ord(c) + i = i+1 + if op >= HAVE_ARGUMENT: + i = i+2 + if op == opmap['JUMP_ABSOLUTE']: + replaced = True + code[orig_i] = chr(opmap['JUMP_ABSOLUTE_UNROLL']) + if not replaced: + raise TypeError("function %s does not contain a loop" % func) + new_codestring = "".join(code) + code = func.func_code + new_code = CodeType(code.co_argcount, code.co_nlocals, code.co_stacksize, + code.co_flags, new_codestring, code.co_consts, code.co_names, + code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, + code.co_lnotab, code.co_freevars, code.co_cellvars) + f = FunctionType(new_code, func.func_globals, func.func_name, + func.func_defaults, func.func_closure) + if func.func_dict: + f.func_dict = {} + f.func_dict.update(func.func_dict) + f.func_doc = func.func_doc + return f diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py --- a/pypy/module/__pypy__/test/test_magic.py +++ b/pypy/module/__pypy__/test/test_magic.py @@ -60,3 +60,22 @@ pass a = A() assert _promote(a) is a + + def test_unroll_safe(self): + from __pypy__ import _unroll_safe + import opcode + def f(x): + r = 0 + for i in range(x): + r += i + return r + assert chr(opcode.opmap['JUMP_ABSOLUTE_UNROLL']) not in f.func_code.co_code + f1 = _unroll_safe(f) + assert chr(opcode.opmap['JUMP_ABSOLUTE_UNROLL']) in f1.func_code.co_code + assert f(10) == f1(10) + + def decorate_no_loop(): + @_unroll_safe + def f(x): + pass + raises(TypeError, decorate_no_loop) From pypy.commits at gmail.com Mon Jun 6 14:11:04 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 11:11:04 -0700 (PDT) Subject: [pypy-commit] pypy default: Backout 4b396d7cbc4c Message-ID: <5755bcb8.0249c20a.76cab.ffff917a@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84978:f0d3545fa2e3 Date: 2016-06-06 19:07 +0100 http://bitbucket.org/pypy/pypy/changeset/f0d3545fa2e3/ Log: Backout 4b396d7cbc4c diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,7 +1,6 @@ import py import pytest - at pytest.mark.trylast def pytest_configure(config): from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) From pypy.commits at gmail.com Mon Jun 6 14:11:02 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 11:11:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Backout 9f49da4855da Message-ID: <5755bcb6.a75ec20a.da57a.ffffa17b@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84977:4c08a181c86a Date: 2016-06-06 19:05 +0100 http://bitbucket.org/pypy/pypy/changeset/4c08a181c86a/ Log: Backout 9f49da4855da diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,5 +1,5 @@ -import pytest from rpython.tool import udir +from pypy.conftest import option from pypy.interpreter.gateway import interp2app def check_no_w_locals(space, w_frame): @@ -11,7 +11,7 @@ space = cls.space cls.w_udir = cls.space.wrap(str(udir.udir)) cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) - if not pytest.config.option.runappdirect: + if not option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): return f() diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -1,4 +1,5 @@ import py +from pypy import conftest from pypy.interpreter import gateway from rpython.rlib.jit import non_virtual_ref, vref_None diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -89,7 +89,8 @@ class AppTestWrapper: def setup_class(cls): "Run test_various_depths() when we are run with 'pypy py.test -A'." - if not py.test.config.option.runappdirect: + from pypy.conftest import option + if not option.runappdirect: py.test.skip("meant only for -A run") def _setup(): diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -516,11 +516,12 @@ assert s == 'bar\n' def test_flush_at_exit(): + from pypy import conftest from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(py.test.config.option) + config = make_config(conftest.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): f = open(tmpfile, 'w') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -222,10 +222,14 @@ assert not closed[0] # flush() called before file closed os.close(fd) -def test_flush_at_exit(space): +def test_flush_at_exit(): + from pypy import conftest + from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') + config = make_config(conftest.option) + space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): import io f = io.open(tmpfile, 'w', encoding='ascii') @@ -237,7 +241,12 @@ assert tmpfile.read() == '42' -def test_flush_at_exit_IOError_and_ValueError(space): +def test_flush_at_exit_IOError_and_ValueError(): + from pypy import conftest + from pypy.tool.option import make_config, make_objspace + + config = make_config(conftest.option) + space = make_objspace(config) space.appexec([], """(): import io class MyStream(io.IOBase): diff --git a/pypy/module/_rawffi/test/test_tracker.py b/pypy/module/_rawffi/test/test_tracker.py --- a/pypy/module/_rawffi/test/test_tracker.py +++ b/pypy/module/_rawffi/test/test_tracker.py @@ -1,5 +1,5 @@ import py -from pytest import option +from pypy.conftest import option from pypy.module._rawffi.tracker import Tracker @@ -44,3 +44,4 @@ def teardown_class(cls): Tracker.DO_TRACING = False + diff --git a/pypy/module/gc/test/test_referents.py b/pypy/module/gc/test/test_referents.py --- a/pypy/module/gc/test/test_referents.py +++ b/pypy/module/gc/test/test_referents.py @@ -1,4 +1,4 @@ -from pytest import option +from pypy.conftest import option class AppTestReferents(object): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -13,7 +13,7 @@ from pypy.module.imp import importing -from pytest import config +from pypy import conftest def setuppkg(pkgname, **entries): p = udir.join('impsubdir') @@ -106,7 +106,7 @@ # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") - if config.option.runappdirect: + if conftest.option.runappdirect: import marshal, stat, struct, os, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) @@ -168,7 +168,7 @@ } def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class @@ -1494,6 +1494,8 @@ spaceconfig = dict(usemodules=['thread', 'time']) def setup_class(cls): + #if not conftest.option.runappdirect: + # py.test.skip("meant as an -A test") tmpfile = udir.join('test_multithreaded_imp.py') tmpfile.write('''if 1: x = 666 diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -1,4 +1,4 @@ -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy import constants as NPY @@ -7,7 +7,7 @@ @classmethod def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: import sys if '__pypy__' not in sys.builtin_module_names: import numpy diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -2,6 +2,7 @@ import sys +from pypy.conftest import option from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest @@ -22,13 +23,13 @@ # special values testing if isnan(a): if isnan(b): - return True, '' + return True,'' raise AssertionError(msg + '%r should be nan' % (b,)) if isinf(a): if a == b: - return True, '' - raise AssertionError(msg + 'finite result where infinity expected: ' + return True,'' + raise AssertionError(msg + 'finite result where infinity expected: '+ \ 'expected %r, got %r' % (a, b)) # if both a and b are zero, check whether they have the same sign @@ -38,7 +39,7 @@ if not a and not b: # only check it if we are running on top of CPython >= 2.6 if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b): - raise AssertionError(msg + + raise AssertionError( msg + \ 'zero has wrong sign: expected %r, got %r' % (a, b)) # if a-b overflows, or b is infinite, return False. Again, in @@ -95,6 +96,7 @@ cls.w_testcases128 = cls.space.wrap(list(parse_testfile(fname128))) cls.w_testcases64 = cls.space.wrap(list(parse_testfile(fname64))) + cls.w_runAppDirect = cls.space.wrap(option.runappdirect) cls.w_isWindows = cls.space.wrap(os.name == 'nt') if cls.runappdirect: diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1,11 +1,11 @@ -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest from pypy.interpreter.gateway import interp2app class BaseAppTestDtypes(BaseNumpyAppTest): def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if config.option.runappdirect: + if option.runappdirect: import platform bits, linkage = platform.architecture() ptr_size = int(bits[:-3]) // 8 @@ -1088,7 +1088,7 @@ spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) - if config.option.runappdirect: + if option.runappdirect: cls.w_test_for_core_internal = cls.space.wrap(True) else: cls.w_test_for_core_internal = cls.space.wrap(False) @@ -1515,7 +1515,7 @@ else: assert stor2[1] == '\x01' assert stor2[0] == '\x00' - if config.option.runappdirect: + if option.runappdirect: cls.w_check_non_native = lambda *args : None else: cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2,7 +2,7 @@ import py import sys -from pytest import config +from pypy.conftest import option from pypy.module.micronumpy.appbridge import get_appbridge_cache from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk from pypy.module.micronumpy.ndarray import W_NDimArray @@ -3850,7 +3850,7 @@ class AppTestRepr(BaseNumpyAppTest): def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: py.test.skip("Can't be run directly.") BaseNumpyAppTest.setup_class.im_func(cls) cache = get_appbridge_cache(cls.space) @@ -3867,7 +3867,7 @@ assert repr(array(1.5).real) == "array(1.5)" def teardown_class(cls): - if config.option.runappdirect: + if option.runappdirect: return cache = get_appbridge_cache(cls.space) cache.w_array_repr = cls.old_array_repr @@ -4343,7 +4343,7 @@ class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): - if config.option.runappdirect and '__pypy__' not in sys.builtin_module_names: + if option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -1,9 +1,14 @@ from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest +from pypy.conftest import option class AppTestObjectDtypes(BaseNumpyAppTest): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) + def test_scalar_from_object(self): from numpy import array import sys diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py --- a/pypy/module/micronumpy/test/test_support_app.py +++ b/pypy/module/micronumpy/test/test_support_app.py @@ -3,12 +3,11 @@ import py from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest -from pytest import config +from pypy.conftest import option class AppTestSupport(BaseNumpyAppTest): def setup_class(cls): - if (config.option.runappdirect and - '__pypy__' not in sys.builtin_module_names): + if option.runappdirect and '__pypy__' not in sys.builtin_module_names: py.test.skip("pypy only test") BaseNumpyAppTest.setup_class.im_func(cls) diff --git a/pypy/module/test_lib_pypy/support.py b/pypy/module/test_lib_pypy/support.py --- a/pypy/module/test_lib_pypy/support.py +++ b/pypy/module/test_lib_pypy/support.py @@ -1,6 +1,6 @@ import py -from pytest import config +from pypy.conftest import option from pypy.interpreter.error import OperationError def import_lib_pypy(space, name, skipmsg=None): @@ -9,7 +9,7 @@ Raises a pytest Skip on ImportError if a skip message was specified. """ - if config.option.runappdirect: + if option.runappdirect: try: mod = __import__('lib_pypy.' + name) except ImportError as e: diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,9 +1,9 @@ -from pytest import config +from pypy import conftest class AppTestBytesArray: def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(config.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) def test_basics(self): b = bytearray() diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -1,4 +1,5 @@ -from pytest import config +from __future__ import with_statement +from pypy.conftest import option class AppTestObject: @@ -6,13 +7,13 @@ from pypy.interpreter import gateway import sys - cpython_behavior = (not config.option.runappdirect + cpython_behavior = (not option.runappdirect or not hasattr(sys, 'pypy_translation_info')) space = cls.space cls.w_cpython_behavior = space.wrap(cpython_behavior) cls.w_cpython_version = space.wrap(tuple(sys.version_info)) - cls.w_appdirect = space.wrap(config.option.runappdirect) + cls.w_appdirect = space.wrap(option.runappdirect) cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info')) def w_unwrap_wrap_unicode(space, w_obj): diff --git a/pypy/objspace/std/test/test_proxy_usercreated.py b/pypy/objspace/std/test/test_proxy_usercreated.py --- a/pypy/objspace/std/test/test_proxy_usercreated.py +++ b/pypy/objspace/std/test/test_proxy_usercreated.py @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app from pypy.objspace.std.transparent import register_proxyable -from pytest import config +from pypy.conftest import option class W_Wrapped(W_Root): @@ -25,7 +25,7 @@ class AppTestProxyNewtype(AppProxy): def setup_class(cls): - if config.option.runappdirect: + if option.runappdirect: py.test.skip("Impossible to run on appdirect") AppProxy.setup_class.im_func(cls) cls.w_wrapped = cls.space.wrap(W_Wrapped()) diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py --- a/pypy/objspace/test/test_binop_overriding.py +++ b/pypy/objspace/test/test_binop_overriding.py @@ -1,5 +1,5 @@ # test about the binop operation rule, see issue 412 -from pytest import config +from pypy.conftest import option class AppTestBinopCombinations: @@ -83,7 +83,7 @@ return Base, do_test """) cls.w_helpers = w_helpers - cls.w_appdirect = cls.space.wrap(config.option.runappdirect) + cls.w_appdirect = cls.space.wrap(option.runappdirect) def test_overriding_base_binop_explict(self): class MulBase(object): diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -184,6 +184,7 @@ source = str(source).strip() except py.error.ENOENT: source = None + from pypy import conftest if source and py.test.config._assertstate.mode != "off": msg = interpret(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -3,13 +3,14 @@ from rpython.config.config import ConflictConfigError from pypy.tool.option import make_config, make_objspace from pypy.tool.pytest import appsupport +from pypy.conftest import option _SPACECACHE={} def gettestobjspace(**kwds): """ helper for instantiating and caching space's for testing. """ try: - config = make_config(py.test.config.option,**kwds) + config = make_config(option,**kwds) except ConflictConfigError as e: # this exception is typically only raised if a module is not available. # in this case the test should be skipped @@ -18,7 +19,7 @@ try: return _SPACECACHE[key] except KeyError: - if getattr(py.test.config.option, 'runappdirect', None): + if getattr(option, 'runappdirect', None): return TinyObjSpace(**kwds) space = maketestobjspace(config) _SPACECACHE[key] = space @@ -26,7 +27,7 @@ def maketestobjspace(config=None): if config is None: - config = make_config(py.test.config.option) + config = make_config(option) if config.objspace.usemodules.thread: config.translation.thread = True space = make_objspace(config) From pypy.commits at gmail.com Mon Jun 6 14:11:05 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 11:11:05 -0700 (PDT) Subject: [pypy-commit] pypy default: Backout bd58f90 Message-ID: <5755bcb9.8b371c0a.ae99c.52d2@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84979:b9e345296b3c Date: 2016-06-06 19:10 +0100 http://bitbucket.org/pypy/pypy/changeset/b9e345296b3c/ Log: Backout bd58f90 diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -23,10 +23,11 @@ def pytest_report_header(): return "pytest-%s from %s" % (pytest.__version__, pytest.__file__) +def pytest_addhooks(pluginmanager): + from rpython.conftest import LeakFinder + pluginmanager.register(LeakFinder()) + def pytest_configure(config): - if not config.option.runappdirect: - from rpython.conftest import LeakFinder - config.pluginmanager.register(LeakFinder()) global option option = config.option From pypy.commits at gmail.com Mon Jun 6 14:23:55 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 11:23:55 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove useless check Message-ID: <5755bfbb.442cc20a.fc766.ffffa0b5@mx.google.com> Author: Ronan Lamy Branch: Changeset: r84980:d3a2481ebbdf Date: 2016-06-06 19:23 +0100 http://bitbucket.org/pypy/pypy/changeset/d3a2481ebbdf/ Log: Remove useless check diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -164,13 +164,6 @@ __multicall__.execute() -def pytest_runtest_teardown(__multicall__, item): - __multicall__.execute() - - if 'pygame' in sys.modules: - assert option.view, ("should not invoke Pygame " - "if conftest.option.view is False") - class PyPyClassCollector(py.test.collect.Class): # All pypy Test classes have a "space" member. From pypy.commits at gmail.com Mon Jun 6 14:30:31 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 06 Jun 2016 11:30:31 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: Backed out changeset: 4b396d7cbc4c Message-ID: <5755c147.426dc20a.c03af.50c4@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r84981:b47d7a38b9c2 Date: 2016-06-06 21:28 +0300 http://bitbucket.org/pypy/pypy/changeset/b47d7a38b9c2/ Log: Backed out changeset: 4b396d7cbc4c diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,7 +1,6 @@ import py import pytest - at pytest.mark.trylast def pytest_configure(config): from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) From pypy.commits at gmail.com Mon Jun 6 15:01:33 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 12:01:33 -0700 (PDT) Subject: [pypy-commit] cffi default: Rewrite the code more clearly, and fix the wrong comment Message-ID: <5755c88d.d21b1c0a.50e8b.6f81@mx.google.com> Author: Armin Rigo Branch: Changeset: r2716:2acec12a75bb Date: 2016-06-06 21:02 +0200 http://bitbucket.org/cffi/cffi/changeset/2acec12a75bb/ Log: Rewrite the code more clearly, and fix the wrong comment diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -406,11 +406,13 @@ ... def process(self): - lib.process_document(lib.my_callback, # the callback - ffi.new_handle(self), # 'void *data' + h = ffi.new_handle(self) + lib.process_document(lib.my_callback, # the callback + h, # 'void *data' args...) - # 'self' stay alive at least until here, which means that - # the ffi.from_handle() done in my_callback() are safe + # 'h' stay alive until here, which means that the + # ffi.from_handle() done in my_callback() during + # the call to process_document() are safe def callback(self, arg1, arg2): ... From pypy.commits at gmail.com Mon Jun 6 15:03:40 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Jun 2016 12:03:40 -0700 (PDT) Subject: [pypy-commit] cffi default: english attempts Message-ID: <5755c90c.92981c0a.7472.6bab@mx.google.com> Author: Armin Rigo Branch: Changeset: r2717:a290f4a53c4d Date: 2016-06-06 21:04 +0200 http://bitbucket.org/cffi/cffi/changeset/a290f4a53c4d/ Log: english attempts diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -410,9 +410,9 @@ lib.process_document(lib.my_callback, # the callback h, # 'void *data' args...) - # 'h' stay alive until here, which means that the + # 'h' stays alive until here, which means that the # ffi.from_handle() done in my_callback() during - # the call to process_document() are safe + # the call to process_document() is safe def callback(self, arg1, arg2): ... From pypy.commits at gmail.com Mon Jun 6 17:07:50 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 06 Jun 2016 14:07:50 -0700 (PDT) Subject: [pypy-commit] pypy s390x-5.3-catchup: (s390x) impl uint_mul_high, removed uint_floordiv, int_floordiv and int_mod Message-ID: <5755e626.4e0b1c0a.8ba14.ffffa1c2@mx.google.com> Author: Richard Plangger Branch: s390x-5.3-catchup Changeset: r84982:9a37908f3ad1 Date: 2016-06-06 23:06 +0200 http://bitbucket.org/pypy/pypy/changeset/9a37908f3ad1/ Log: (s390x) impl uint_mul_high, removed uint_floordiv, int_floordiv and int_mod diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -439,7 +439,7 @@ prepare_int_lshift = helper.prepare_binary_op prepare_int_rshift = helper.prepare_binary_op prepare_uint_rshift = helper.prepare_binary_op - prepare_uint_mul_high = helper.prepare_binary_op + prepare_uint_mul_high = helper.prepare_int_mul_ovf prepare_int_add_ovf = helper.prepare_binary_op prepare_int_sub_ovf = helper.prepare_binary_op diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -29,6 +29,7 @@ 'MGHI': ('ri', ['\xA7','\x0D']), 'MSGFI': ('ril', ['\xC2','\x00']), 'MLGR': ('rre', ['\xB9','\x86'], 'eo,r'), + 'MLG': ('rxy', ['\xE3','\x86'], 'eo,bid'), # div/mod 'DSGR': ('rre', ['\xB9','\x0D'], 'eo,r'), 'DSG': ('rxy', ['\xE3','\x0D'], 'eo,bidl'), diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -160,11 +160,15 @@ omc.BRC(c.ANY, l.imm(label_end - jmp_neither_lqlr_overflow)) omc.overwrite() - emit_int_floordiv = gen_emit_div_mod('DSGR', 'DSG') - emit_uint_floordiv = gen_emit_div_mod('DLGR', 'DLG') - # NOTE division sets one register with the modulo value, thus - # the regalloc ensures the right register survives. - emit_int_mod = gen_emit_div_mod('DSGR', 'DSG') + def emit_uint_mul_high(self, op, arglocs, regalloc): + r0, _, a1 = arglocs + # _ carries the value, contents of r0 are ignored + assert not r0.is_imm() + assert not a1.is_imm() + if a1.is_core_reg(): + self.mc.MLGR(r0, a1) + else: + self.mc.MLG(r0, a1) def emit_int_invert(self, op, arglocs, regalloc): l0, = arglocs diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -733,9 +733,6 @@ prepare_int_sub_ovf = helper.prepare_int_sub prepare_int_mul = helper.prepare_int_mul prepare_int_mul_ovf = helper.prepare_int_mul_ovf - prepare_int_floordiv = helper.prepare_int_div - prepare_uint_floordiv = helper.prepare_int_div - prepare_int_mod = helper.prepare_int_mod prepare_nursery_ptr_increment = prepare_int_add prepare_int_and = helper.prepare_int_logic @@ -746,6 +743,18 @@ prepare_int_lshift = helper.prepare_int_shift prepare_uint_rshift = helper.prepare_int_shift + def prepare_uint_mul_high(self, op): + a0 = op.getarg(0) + a1 = op.getarg(1) + if a0.is_constant(): + a0, a1 = a1, a0 + if helper.check_imm32(a1): + l1 = self.ensure_reg(a1) + else: + l1 = self.ensure_reg_or_pool(a1) + lr,lq = self.rm.ensure_even_odd_pair(a0, op, bind_first=True) + return [lr, lq, l1] + prepare_int_le = helper.generate_cmp_op() prepare_int_lt = helper.generate_cmp_op() prepare_int_ge = helper.generate_cmp_op() From pypy.commits at gmail.com Mon Jun 6 17:10:25 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 06 Jun 2016 14:10:25 -0700 (PDT) Subject: [pypy-commit] pypy s390x-5.3-catchup: (s390x) killed some test that where specific to the three removed resoperations Message-ID: <5755e6c1.891ec20a.11f71.ffffcbf6@mx.google.com> Author: Richard Plangger Branch: s390x-5.3-catchup Changeset: r84983:49d162b16626 Date: 2016-06-06 23:09 +0200 http://bitbucket.org/pypy/pypy/changeset/49d162b16626/ Log: (s390x) killed some test that where specific to the three removed resoperations diff --git a/rpython/jit/backend/zarch/test/test_int.py b/rpython/jit/backend/zarch/test/test_int.py --- a/rpython/jit/backend/zarch/test/test_int.py +++ b/rpython/jit/backend/zarch/test/test_int.py @@ -35,41 +35,13 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail == finishdescr # ensures that guard is not taken! - def test_double_evenodd_pair(self): - code = """ - [i0] - i1 = int_floordiv(i0, 2) - i2 = int_floordiv(i0, 3) - i3 = int_floordiv(i0, 4) - i4 = int_floordiv(i0, 5) - i5 = int_floordiv(i0, 6) - i6 = int_floordiv(i0, 7) - i7 = int_floordiv(i0, 8) - i8 = int_le(i1, 0) - guard_true(i8) [i1,i2,i3,i4,i5,i6,i7] - finish(i0, descr=faildescr) - """ - # the guard forces 3 spills because after 4 divisions - # all even slots of the managed registers are full - loop = parse(code, namespace={'faildescr': BasicFinalDescr(1)}) - looptoken = JitCellToken() - self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - deadframe = self.cpu.execute_token(looptoken, 100) - fail = self.cpu.get_latest_descr(deadframe) - for i in range(2,9): - assert self.cpu.get_int_value(deadframe, i-2) == 100//i - - - @py.test.mark.parametrize('value', [2,3,15,2**16]) def test_evenodd_pair_extensive(self, value): instrs = [] failargs = [] values = [] j = 0 - mapping = (('int_floordiv',lambda x,y: x // y), - ('int_mod', lambda x,y: x % y), - ('int_mul_ovf', lambda x,y: x * y)) + mapping = (('int_mul_ovf', lambda x,y: x * y),) for i in range(20): name, func = mapping[j] instrs.append("i{d} = {i}(i0, {d})".format(d=i+1, i=name)) From pypy.commits at gmail.com Mon Jun 6 18:23:42 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 06 Jun 2016 15:23:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: prepare vars for build_map_unpack_with_call Message-ID: <5755f7ee.04251c0a.b4768.0c8c@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r84984:86822511a36a Date: 2016-06-07 00:22 +0200 http://bitbucket.org/pypy/pypy/changeset/86822511a36a/ Log: prepare vars for build_map_unpack_with_call diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1351,9 +1351,14 @@ itemcount -= 1 self.pushvalue(w_sum) + #TODO + #get intersection, store as setentry def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): w_sum = self.space.newset() + num_maps = itemcount & 0xff + function_location = (itemcount >> 8) & 0xff for i in range(itemcount, 0, -1): + arg = self.space.peek(i) self.space.call_method(w_sum, 'update', self.space.peek(i)) while itemcount != 0: self.popvalue() From pypy.commits at gmail.com Mon Jun 6 18:36:21 2016 From: pypy.commits at gmail.com (pjenvey) Date: Mon, 06 Jun 2016 15:36:21 -0700 (PDT) Subject: [pypy-commit] pypy py3k: seems to work now Message-ID: <5755fae5.6a43c20a.14cb0.ffffc323@mx.google.com> Author: Philip Jenvey Branch: py3k Changeset: r84985:a02677ec50cc Date: 2016-06-06 15:35 -0700 http://bitbucket.org/pypy/pypy/changeset/a02677ec50cc/ Log: seems to work now diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test/test_structseq.py --- a/pypy/module/cpyext/test/test_structseq.py +++ b/pypy/module/cpyext/test/test_structseq.py @@ -6,7 +6,6 @@ class AppTestStructSeq(AppTestCpythonExtensionBase): def test_StructSeq(self): - skip("XXX: https://bugs.pypy.org/issue1557") module = self.import_extension('foo', prologue=""" #include From pypy.commits at gmail.com Mon Jun 6 19:55:48 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 16:55:48 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: Kill PyPyClassCollector.setup() Message-ID: <57560d84.e4b3c20a.35934.11a0@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r84986:775591186dfd Date: 2016-06-07 00:39 +0100 http://bitbucket.org/pypy/pypy/changeset/775591186dfd/ Log: Kill PyPyClassCollector.setup() diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -160,20 +160,15 @@ if spaceconfig is not None: from pypy.tool.pytest.objspace import gettestobjspace appclass.obj.space = gettestobjspace(**spaceconfig) + else: + appclass.obj.space = LazyObjSpaceGetter() appclass.obj.runappdirect = option.runappdirect __multicall__.execute() class PyPyClassCollector(py.test.collect.Class): - # All pypy Test classes have a "space" member. - def setup(self): - cls = self.obj - if not hasattr(cls, 'spaceconfig'): - cls.space = LazyObjSpaceGetter() - else: - assert hasattr(cls, 'space') # set by pytest_runtest_setup - super(PyPyClassCollector, self).setup() + pass def pytest_ignore_collect(path): From pypy.commits at gmail.com Mon Jun 6 19:55:50 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 16:55:50 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: Remove obsolete check Message-ID: <57560d86.071d1c0a.51556.ffffd6b8@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r84987:0b983b71075f Date: 2016-06-07 00:55 +0100 http://bitbucket.org/pypy/pypy/changeset/0b983b71075f/ Log: Remove obsolete check diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py --- a/pypy/tool/pytest/inttest.py +++ b/pypy/tool/pytest/inttest.py @@ -33,15 +33,6 @@ except OperationError as e: check_keyboard_interrupt(e) raise - except Exception as e: - cls = e.__class__ - while cls is not Exception: - if cls.__name__ == 'DistutilsPlatformError': - from distutils.errors import DistutilsPlatformError - if isinstance(e, DistutilsPlatformError): - py.test.skip('%s: %s' % (e.__class__.__name__, e)) - cls = cls.__bases__[0] - raise class IntInstanceCollector(py.test.collect.Instance): From pypy.commits at gmail.com Mon Jun 6 20:35:08 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 06 Jun 2016 17:35:08 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: Kill IntTestFunction.runtest() since it doesn't seem to do anything ever Message-ID: <575616bc.073f1c0a.734a7.ffffd1eb@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r84988:29830f8f8a68 Date: 2016-06-07 01:34 +0100 http://bitbucket.org/pypy/pypy/changeset/29830f8f8a68/ Log: Kill IntTestFunction.runtest() since it doesn't seem to do anything ever diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py --- a/pypy/tool/pytest/inttest.py +++ b/pypy/tool/pytest/inttest.py @@ -3,22 +3,9 @@ # Most pypy tests are of this kind. import py -import sys -from pypy.interpreter.error import OperationError from pypy.conftest import PyPyClassCollector -def check_keyboard_interrupt(e): - # we cannot easily convert w_KeyboardInterrupt to KeyboardInterrupt - # in general without a space -- here is an approximation - try: - if e.w_type.name == 'KeyboardInterrupt': - tb = sys.exc_info()[2] - raise KeyboardInterrupt, KeyboardInterrupt(), tb - except AttributeError: - pass - - marker = py.test.mark.interplevel @@ -27,13 +14,6 @@ super(IntTestFunction, self).__init__(*args, **kwargs) self._request.applymarker(marker) - def runtest(self): - try: - super(IntTestFunction, self).runtest() - except OperationError as e: - check_keyboard_interrupt(e) - raise - class IntInstanceCollector(py.test.collect.Instance): Function = IntTestFunction From pypy.commits at gmail.com Tue Jun 7 08:17:31 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 05:17:31 -0700 (PDT) Subject: [pypy-commit] pypy s390x-5.3-catchup: (s390x) removed two custom s390x tests that was using floordiv and mod Message-ID: <5756bb5b.872b1c0a.973f2.ffffb5e5@mx.google.com> Author: Richard Plangger Branch: s390x-5.3-catchup Changeset: r84989:d793a0aecdb5 Date: 2016-06-07 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/d793a0aecdb5/ Log: (s390x) removed two custom s390x tests that was using floordiv and mod diff --git a/rpython/jit/backend/zarch/test/test_regalloc.py b/rpython/jit/backend/zarch/test/test_regalloc.py --- a/rpython/jit/backend/zarch/test/test_regalloc.py +++ b/rpython/jit/backend/zarch/test/test_regalloc.py @@ -146,128 +146,3 @@ assert cpu.get_int_value(deadframe, 0) == 0 assert cpu.get_int_value(deadframe, 1) == -1000 -def test_bug_0(): - cpu, deadframe = run([-13, 10, 10, 8, -8, -16, -18, 46, -12, 26], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_gt(i3, -48) - i12 = int_xor(i8, i1) - i13 = int_gt(i6, -9) - i14 = int_le(i13, i2) - i15 = int_le(i11, i5) - i16 = uint_ge(i13, i13) - i17 = int_or(i9, -23) - i18 = int_lt(i10, i13) - i19 = int_or(i15, i5) - i20 = int_xor(i17, 54) - i21 = int_mul(i8, i10) - i22 = int_or(i3, i9) - i41 = int_and(i11, -4) - i42 = int_or(i41, 1) - i23 = int_mod(i12, i42) - i24 = int_is_true(i6) - i25 = uint_rshift(i15, 6) - i26 = int_or(-4, i25) - i27 = int_invert(i8) - i28 = int_sub(-113, i11) - i29 = int_neg(i7) - i30 = int_neg(i24) - i31 = int_floordiv(i3, 53) - i32 = int_mul(i28, i27) - i43 = int_and(i18, -4) - i44 = int_or(i43, 1) - i33 = int_mod(i26, i44) - i34 = int_or(i27, i19) - i35 = uint_lt(i13, 1) - i45 = int_and(i21, 31) - i36 = int_rshift(i21, i45) - i46 = int_and(i20, 31) - i37 = uint_rshift(i4, i46) - i38 = uint_gt(i33, -11) - i39 = int_neg(i7) - i40 = int_gt(i24, i32) - i99 = same_as_i(0) - guard_true(i99) [i40, i36, i37, i31, i16, i34, i35, i23, i22, i29, i14, i39, i30, i38] - finish(42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 0 - assert cpu.get_int_value(deadframe, 2) == 0 - assert cpu.get_int_value(deadframe, 3) == 0 - assert cpu.get_int_value(deadframe, 4) == 1 - assert cpu.get_int_value(deadframe, 5) == -7 - assert cpu.get_int_value(deadframe, 6) == 1 - assert cpu.get_int_value(deadframe, 7) == 0 - assert cpu.get_int_value(deadframe, 8) == -2 - assert cpu.get_int_value(deadframe, 9) == 18 - assert cpu.get_int_value(deadframe, 10) == 1 - assert cpu.get_int_value(deadframe, 11) == 18 - assert cpu.get_int_value(deadframe, 12) == -1 - assert cpu.get_int_value(deadframe, 13) == 0 - -def test_bug_1(): - cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_lt(i6, 0) - i41 = int_and(i3, 31) - i12 = int_rshift(i3, i41) - i13 = int_neg(i2) - i14 = int_add(i11, i7) - i15 = int_or(i3, i2) - i16 = int_or(i12, i12) - i17 = int_ne(i2, i5) - i42 = int_and(i5, 31) - i18 = uint_rshift(i14, i42) - i43 = int_and(i14, 31) - i19 = int_lshift(7, i43) - i20 = int_neg(i19) - i21 = int_mod(i3, 1) - i22 = uint_ge(i15, i1) - i44 = int_and(i16, 31) - i23 = int_lshift(i8, i44) - i24 = int_is_true(i17) - i45 = int_and(i5, 31) - i25 = int_lshift(i14, i45) - i26 = int_lshift(i5, 17) - i27 = int_eq(i9, i15) - i28 = int_ge(0, i6) - i29 = int_neg(i15) - i30 = int_neg(i22) - i31 = int_add(i7, i16) - i32 = uint_lt(i19, i19) - i33 = int_add(i2, 1) - i34 = int_neg(i5) - i35 = int_add(i17, i24) - i36 = uint_lt(2, i16) - i37 = int_neg(i9) - i38 = int_gt(i4, i11) - i39 = int_lt(i27, i22) - i40 = int_neg(i27) - i99 = same_as_i(0) - guard_true(i99) [i40, i10, i36, i26, i13, i30, i21, i33, i18, i25, i31, i32, i28, i29, i35, i38, i20, i39, i34, i23, i37] - finish(-42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 8 - assert cpu.get_int_value(deadframe, 2) == 1 - assert cpu.get_int_value(deadframe, 3) == 131072 - assert cpu.get_int_value(deadframe, 4) == 20 - assert cpu.get_int_value(deadframe, 5) == -1 - assert cpu.get_int_value(deadframe, 6) == 0 - assert cpu.get_int_value(deadframe, 7) == -19 - assert cpu.get_int_value(deadframe, 8) == 6 - assert cpu.get_int_value(deadframe, 9) == 26 - assert cpu.get_int_value(deadframe, 10) == 12 - assert cpu.get_int_value(deadframe, 11) == 0 - assert cpu.get_int_value(deadframe, 12) == 0 - assert cpu.get_int_value(deadframe, 13) == 2 - assert cpu.get_int_value(deadframe, 14) == 2 - assert cpu.get_int_value(deadframe, 15) == 1 - assert cpu.get_int_value(deadframe, 16) == -57344 - assert cpu.get_int_value(deadframe, 17) == 1 - assert cpu.get_int_value(deadframe, 18) == -1 - if WORD == 4: - assert cpu.get_int_value(deadframe, 19) == -2147483648 - elif WORD == 8: - assert cpu.get_int_value(deadframe, 19) == 19327352832 - assert cpu.get_int_value(deadframe, 20) == -49 - From pypy.commits at gmail.com Tue Jun 7 08:26:57 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 05:26:57 -0700 (PDT) Subject: [pypy-commit] pypy z196-support: merged default Message-ID: <5756bd91.08371c0a.7877d.ffffbcc2@mx.google.com> Author: Richard Plangger Branch: z196-support Changeset: r84990:e7e9bb78a679 Date: 2016-06-07 14:24 +0200 http://bitbucket.org/pypy/pypy/changeset/e7e9bb78a679/ Log: merged default diff too long, truncating to 2000 out of 14007 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -22,3 +22,6 @@ bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1 +80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 +40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -43,17 +43,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -93,9 +93,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -104,17 +104,20 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -122,13 +125,13 @@ Simon Cross Edd Barrett Andreas Stührk + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -140,7 +143,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -156,11 +158,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -171,9 +175,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -183,8 +187,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -208,11 +210,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -228,7 +230,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -270,8 +271,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -295,9 +297,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -834,54 +834,63 @@ c2pread, c2pwrite = None, None errread, errwrite = None, None + ispread = False if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = _subprocess.CreatePipe(None, 0) + ispread = True elif stdin == PIPE: p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) + ispread = True elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) + p2cread = self._make_inheritable(p2cread, ispread) # We just duplicated the handle, it has to be closed at the end to_close.add(p2cread) if stdin == PIPE: to_close.add(p2cwrite) + ispwrite = False if stdout is None: c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stdout == PIPE: c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) + c2pwrite = self._make_inheritable(c2pwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(c2pwrite) if stdout == PIPE: to_close.add(c2pread) + ispwrite = False if stderr is None: errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) if errwrite is None: _, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == PIPE: errread, errwrite = _subprocess.CreatePipe(None, 0) + ispwrite = True elif stderr == STDOUT: - errwrite = c2pwrite.handle # pass id to not close it + errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) + errwrite = self._make_inheritable(errwrite, ispwrite) # We just duplicated the handle, it has to be closed at the end to_close.add(errwrite) if stderr == PIPE: @@ -892,13 +901,14 @@ errread, errwrite), to_close - def _make_inheritable(self, handle): + def _make_inheritable(self, handle, close=False): """Return a duplicate of handle, which is inheritable""" dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), handle, _subprocess.GetCurrentProcess(), 0, 1, _subprocess.DUPLICATE_SAME_ACCESS) - # If the initial handle was obtained with CreatePipe, close it. - if not isinstance(handle, int): + # PyPy: If the initial handle was obtained with CreatePipe, + # close it. + if close: handle.Close() return dupl diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py --- a/lib-python/2.7/test/test_sys_settrace.py +++ b/lib-python/2.7/test/test_sys_settrace.py @@ -328,8 +328,8 @@ def test_13_genexp(self): if self.using_gc: + gc.enable() test_support.gc_collect() - gc.enable() try: self.run_test(generator_example) # issue1265: if the trace function contains a generator, diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py --- a/lib_pypy/_pypy_interact.py +++ b/lib_pypy/_pypy_interact.py @@ -6,7 +6,7 @@ irc_header = "And now for something completely different" -def interactive_console(mainmodule=None, quiet=False): +def interactive_console(mainmodule=None, quiet=False, future_flags=0): # set sys.{ps1,ps2} just before invoking the interactive interpreter. This # mimics what CPython does in pythonrun.c if not hasattr(sys, 'ps1'): @@ -37,15 +37,17 @@ raise ImportError from pyrepl.simple_interact import run_multiline_interactive_console except ImportError: - run_simple_interactive_console(mainmodule) + run_simple_interactive_console(mainmodule, future_flags=future_flags) else: - run_multiline_interactive_console(mainmodule) + run_multiline_interactive_console(mainmodule, future_flags=future_flags) -def run_simple_interactive_console(mainmodule): +def run_simple_interactive_console(mainmodule, future_flags=0): import code if mainmodule is None: import __main__ as mainmodule console = code.InteractiveConsole(mainmodule.__dict__, filename='') + if future_flags: + console.compile.compiler.flags |= future_flags # some parts of code.py are copied here because it seems to be impossible # to start an interactive console without printing at least one line # of banner diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py --- a/lib_pypy/_pypy_irc_topic.py +++ b/lib_pypy/_pypy_irc_topic.py @@ -224,23 +224,9 @@ va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat """ -from string import ascii_uppercase, ascii_lowercase - def rot13(data): - """ A simple rot-13 encoder since `str.encode('rot13')` was removed from - Python as of version 3.0. It rotates both uppercase and lowercase letters individually. - """ - total = [] - for char in data: - if char in ascii_uppercase: - index = (ascii_uppercase.find(char) + 13) % 26 - total.append(ascii_uppercase[index]) - elif char in ascii_lowercase: - index = (ascii_lowercase.find(char) + 13) % 26 - total.append(ascii_lowercase[index]) - else: - total.append(char) - return "".join(total) + return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else + -13 if 'N'<=c.upper()<='Z' else 0)) for c in data) def some_topic(): import time diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py --- a/lib_pypy/_subprocess.py +++ b/lib_pypy/_subprocess.py @@ -4,6 +4,9 @@ subprocess module on Windows. """ +import sys +if sys.platform != 'win32': + raise ImportError("The '_subprocess' module is only available on Windows") # Declare external Win32 functions diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.6.0 +Version: 1.7.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.6.0" -__version_info__ = (1, 6, 0) +__version__ = "1.7.0" +__version_info__ = (1, 7, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -57,6 +57,12 @@ # define _CFFI_UNUSED_FN /* nothing */ #endif +#ifdef __cplusplus +# ifndef _Bool +# define _Bool bool /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + /********** CPython-specific section **********/ #ifndef PYPY_VERSION diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.6.0" + "\ncompiled with cffi version: 1.7.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -332,8 +332,8 @@ def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. - Note that this is not meant to be used on the built-in types str, - unicode, or bytearray (you can build 'char[]' arrays explicitly) + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -205,9 +205,7 @@ def __nonzero__(self): return bool(self._address) - - def __bool__(self): - return bool(self._address) + __bool__ = __nonzero__ @classmethod def _to_ctypes(cls, value): @@ -465,6 +463,7 @@ else: def __nonzero__(self): return self._value != 0 + __bool__ = __nonzero__ if kind == 'float': @staticmethod diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py --- a/lib_pypy/cffi/commontypes.py +++ b/lib_pypy/cffi/commontypes.py @@ -35,8 +35,11 @@ "you call ffi.set_unicode()" % (commontype,)) else: if commontype == cdecl: - raise api.FFIError("Unsupported type: %r. Please file a bug " - "if you think it should be." % (commontype,)) + raise api.FFIError( + "Unsupported type: %r. Please look at " + "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " + "and file an issue if you think this type should really " + "be supported." % (commontype,)) result, quals = parser.parse_type_and_quals(cdecl) # recursive assert isinstance(result, model.BaseTypeByIdentity) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -814,7 +814,7 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) << 1); /* check that '%s.%s' is " + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' @@ -991,7 +991,7 @@ prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') prnt(' int n = (%s) <= 0;' % (name,)) - prnt(' *o = (unsigned long long)((%s) << 0);' + prnt(' *o = (unsigned long long)((%s) | 0);' ' /* check that %s is an integer */' % (name, name)) if check_value is not None: if check_value > 0: @@ -1250,7 +1250,7 @@ def _emit_bytecode_UnknownIntegerType(self, tp, index): s = ('_cffi_prim_int(sizeof(%s), (\n' - ' ((%s)-1) << 0 /* check that %s is an integer type */\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' ' ) <= 0)' % (tp.name, tp.name, tp.name)) self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py --- a/lib_pypy/pyrepl/simple_interact.py +++ b/lib_pypy/pyrepl/simple_interact.py @@ -43,11 +43,13 @@ return short return text -def run_multiline_interactive_console(mainmodule=None): +def run_multiline_interactive_console(mainmodule=None, future_flags=0): import code if mainmodule is None: import __main__ as mainmodule console = code.InteractiveConsole(mainmodule.__dict__, filename='') + if future_flags: + console.compile.compiler.flags |= future_flags def more_lines(unicodetext): # ooh, look at the hack: diff --git a/pypy/__init__.py b/pypy/__init__.py --- a/pypy/__init__.py +++ b/pypy/__init__.py @@ -1,4 +1,5 @@ -# Empty +import os +pypydir = os.path.realpath(os.path.dirname(__file__)) # XXX Should be empty again, soon. # XXX hack for win64: diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -1,4 +1,4 @@ -import py, pytest, sys, os, textwrap +import py, pytest, sys, textwrap from inspect import isclass # pytest settings @@ -10,8 +10,6 @@ # option = None -pypydir = os.path.realpath(os.path.dirname(__file__)) - def braindead_deindent(self): """monkeypatch that wont end up doing stupid in the python tokenizer""" text = '\n'.join(self.lines) @@ -166,13 +164,6 @@ __multicall__.execute() -def pytest_runtest_teardown(__multicall__, item): - __multicall__.execute() - - if 'pygame' in sys.modules: - assert option.view, ("should not invoke Pygame " - "if conftest.option.view is False") - class PyPyClassCollector(py.test.collect.Class): # All pypy Test classes have a "space" member. diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -70,9 +70,6 @@ bz2 libbz2 -lzma (PyPy3 only) - liblzma - pyexpat libexpat1 @@ -98,11 +95,16 @@ tk tk-dev +lzma (PyPy3 only) + liblzma + +To run untranslated tests, you need the Boehm garbage collector libgc. + On Debian, this is the command to install all build-time dependencies:: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ - tk-dev libgc-dev + tk-dev libgc-dev liblzma-dev For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -13,17 +13,17 @@ Samuele Pedroni Matti Picus Alex Gaynor + Philip Jenvey Brian Kearns - Philip Jenvey + Ronan Lamy Michael Hudson - Ronan Lamy + Manuel Jacob David Schneider - Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo + Richard Plangger Benjamin Peterson - Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen @@ -63,9 +63,9 @@ stian Jan de Mooij Tyler Wade + Vincent Legoll Michael Foord Stephan Diehl - Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -74,31 +74,34 @@ Bruno Gola David Malcolm Jean-Paul Calderone + Mark Young Timo Paulssen Squeaky + Devin Jeanpierre Marius Gedminas Alexandre Fayolle Simon Burton + Stefano Rivera Martin Matusiak Konstantin Lopuhin - Stefano Rivera Wenzhu Man John Witulski Laurence Tratt + Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross + Edd Barrett Andreas Stührk - Edd Barrett + Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -110,7 +113,6 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -126,11 +128,13 @@ Dusty Phillips Lukas Renggli Guenter Jantzen + William Leslie Ned Batchelder Tim Felgentreff Anton Gulenko Amit Regmi Ben Young + Sergey Matyunin Nicolas Chauvat Andrew Durdin Andrew Chambers @@ -141,9 +145,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas - Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila + anatoly techtonik Gabriel Lavoie Olivier Dormond Jared Grubb @@ -153,8 +157,6 @@ Brian Dorsey Victor Stinner Andrews Medina - anatoly techtonik - Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -178,11 +180,11 @@ Alex Perry Vaibhav Sood Alan McIntyre - William Leslie Alexander Sedov Attila Gobi Jasper.Schulz Christopher Pope + Florin Papa Christian Tismer Marc Abramowitz Dan Stromberg @@ -198,7 +200,6 @@ Lukas Vacek Kunal Grover Andrew Dalke - Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -240,8 +241,9 @@ Yury V. Zaytsev Anna Katrina Dominguez Bobby Impollonia - timo at eistee.fritz.box + Vasantha Ganesh K Andrew Thompson + florinpapa Yusei Tahara Aaron Tubbs Ben Darnell @@ -265,9 +267,9 @@ Akira Li Gustavo Niemeyer Stephan Busemann - florinpapa Rafał Gałczyński Matt Bogosian + timo Christian Muirhead Berker Peksag James Lan diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst release-5.0.1.rst @@ -49,6 +50,13 @@ release-0.6 +CPython 3.3 compatible versions +------------------------------- + +.. toctree:: + + release-pypy3.3-v5.2-alpha1.rst + CPython 3.2 compatible versions ------------------------------- diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst whatsnew-4.0.1.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -53,15 +53,17 @@ immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated. -Numpy improvements ------------------- +NumPy rebooted +-------------- -The numpy is rapidly progressing in pypy, so feel free to come to IRC and -ask for proposed topic. A not necesarilly up-to-date `list of topics`_ -is also available. +Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. +Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy +test suite. We could use help analyzing the failures and fixing them either +as patches to upstream NumPy, or as fixes to PyPy. -.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt - +We also are looking for help in how to hijack NumPy dtype conversion and +ufunc calls to allow the JIT to make them fast, using our internal _numpypy +module. Improving the jitviewer ------------------------ diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.0.rst @@ -0,0 +1,193 @@ +============ +PyPy2.7 v5.3 +============ + +We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after +`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3 +compatibility. This new PyPy2.7 release includes further improvements for the +CAPI compatibility layer which we call cpyext. In addtion to complete support +for lxml, we now pass most (more than 90%) of the upstream numpy test suite, +and much of SciPy is supported as well. + +We updated cffi_ to version 1.7 (small changes, documented here_). + +.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html +.. _cffi: https://cffi.readthedocs.org +.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html + +You can download the PyPy2.7 v5.3 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.1 released in April 2016) +========================================================= + +* New features: + + * Merge a major expansion of the C-API support in cpyext, here are some of + the highlights: + + - allow c-snippet tests to be run with -A so we can verify we are compatible + - fix many edge cases exposed by fixing tests to run with -A + - issequence() logic matches cpython + - make PyStringObject and PyUnicodeObject field names compatible with cpython + - add prelminary support for PyDateTime_* + - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, + PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, + PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo + - improve support for PyGILState_Ensure, PyGILState_Release, and thread + primitives, also find a case where CPython will allow thread creation + before PyEval_InitThreads is run, dissallow on PyPy + - create a PyObject-specific list strategy + - rewrite slot assignment for typeobjects + - improve tracking of PyObject to rpython object mapping + - support tp_as_{number, sequence, mapping, buffer} slots + - support ByteArrayObject via the new resizable_list_supporting_raw_ptr + - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's + - fix the signature of PyUFunc_FromFuncAndDataAndSignature + - implement many PyWhatever_FOO() as a macro taking a `void *` + + * CPyExt tweak: instead of "GIL not held when a CPython C extension module + calls PyXxx", we now silently acquire/release the GIL. Helps with + CPython C extension modules that call some PyXxx() functions without + holding the GIL (arguably, they are theorically buggy). + + * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. + It is a more flexible way to make RPython finalizers. Use this mechanism to + clean up handling of ``__del__`` methods, fixing issue #2287 + + * Generalize cpyext old-style buffers to more than just str/buffer, add + support for mmap + + * Support command line -v to trace import statements + + * Add rposix functions for PyPy3.3 support + + * Give super an __init__ and a simple __new__ for CPython compatibility + + * Revive traceviewer, a tool to use pygame to view traces + +* Bug Fixes + + * Fix issue #2277: only special-case two exact lists in zip(), not list + subclasses, because an overridden __iter__() should be called (probably) + + * Fix issue #2226: Another tweak in the incremental GC- this should ensure + that progress in the major GC occurs quickly enough in all cases. + + * Clarify and refactor documentation on http://doc.pypy.org + + * Use "must be unicode, not %T" in unicodedata TypeErrors. + + * Manually reset sys.settrace() and sys.setprofile() when we're done running. + This is not exactly what CPython does, but if we get an exception, unlike + CPython, we call functions from the 'traceback' module, and these would + call more the trace/profile function. That's unexpected and can lead + to more crashes at this point. + + * Use the appropriate tp_dealloc on a subclass of a builtin type, and call + tp_new for a python-sublcass of a C-API type + + * Fix for issue #2285 - rare vmprof segfaults on OS/X + + * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc + + * Fix issue #2311 - grab the `__future__` flags imported in the main script, in + `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* Numpy_: + + * Implement ufunc.outer on numpypy + + * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_ + accordingly) + +* Performance improvements: + + * Use bitstrings to compress lists of descriptors that are attached to an + EffectInfo + + * Remove most of the _ovf, _zer and _val operations from RPython. Kills + quite some code internally, and allows the JIT to do better + optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` + can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly + negative. + + * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())` + + * Use the __builtin_add_overflow built-ins if they are available + + * Rework the way registers are moved/spilled in before_call() + +* Internal refactorings: + + * Refactor code to better support Python3-compatible syntax + + * Document and refactor OperationError -> oefmt + + * Reduce the size of generated C sources during translation by + eliminating many many unused struct declarations (Issue #2281) + + * Remove a number of translation-time options that were not tested and + never used. Also fix a performance bug in the method cache + + * Reduce the size of generated code by using the same function objects in + all generated subclasses + + * Share cpyext Py* function wrappers according to the signature, shrining the + translated libpypy.so by about + + * Compile c snippets with -Werror, and fix warnings it exposed + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html +.. _Numpy: https://bitbucket.org/pypy/numpy +.. _`the repo`: https://bitbucket.org/pypy/numpy + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst @@ -0,0 +1,69 @@ +=================== +PyPy3 v5.2 alpha 1 +=================== + +We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the +first release of PyPy which targets Python 3.3 (3.3.5) compatibility. + +We would like to thank all of the people who donated_ to the `py3k proposal`_ +for supporting the work that went into this and future releases. + +You can download the PyPy3.3 v5.2 alpha 1 release here: + + http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2 + +Highlights +========== + +* Python 3.3.5 support! + + - Being an early alpha release, there are some `missing features`_ such as a + `PEP 393-like space efficient string representation`_ and `known issues`_ + including performance regressions (e.g. issue `#2305`_). The focus for this + release has been updating to 3.3 compatibility. Windows is also not yet + supported. + +* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4). + +What is PyPy? +============== + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT +compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems except Windows + (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Please try it out and let us know what you think. We welcome feedback, we know +you are using PyPy, please tell us about it! + +We'd especially like to thank these people for their contributions to this +release: + +Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey, +Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others. + +Cheers + +The PyPy Team + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation +.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement +.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29 +.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305 +.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip +.. _`dynamic languages`: http://pypyjs.org diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -73,6 +73,8 @@ 'Richard Lancaster':['richardlancaster'], 'William Leslie':['William ML Leslie'], 'Spenser Bauman':['Spenser Andrew Bauman'], + 'Raffael Tfirst':['raffael.tfirst at gmail.com'], + 'timo':['timo at eistee.fritz.box'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,103 +1,7 @@ ========================= -What's new in PyPy 5.1+ +What's new in PyPy2.7 5.3+ ========================= -.. this is a revision shortly after release-5.1 -.. startrev: aa60332382a1 +.. this is a revision shortly after release-pypy2.7-v5.3 +.. startrev: 873218a739f1 -.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046 - -.. branch: gcheader-decl - -Reduce the size of generated C sources. - - -.. branch: remove-objspace-options - -Remove a number of options from the build process that were never tested and -never set. Fix a performance bug in the method cache. - -.. branch: bitstring - -JIT: use bitstrings to compress the lists of read or written descrs -that we attach to EffectInfo. Fixes a problem we had in -remove-objspace-options. - -.. branch: cpyext-for-merge - -Update cpyext C-API support After this branch, we are almost able to support -upstream numpy via cpyext, so we created (yet another) fork of numpy at -github.com/pypy/numpy with the needed changes. Among the significant changes -to cpyext: - - allow c-snippet tests to be run with -A so we can verify we are compatible - - fix many edge cases exposed by fixing tests to run with -A - - issequence() logic matches cpython - - make PyStringObject and PyUnicodeObject field names compatible with cpython - - add prelminary support for PyDateTime_* - - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy, - PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile, - - PyAnySet_CheckExact, PyUnicode_Concat - - improve support for PyGILState_Ensure, PyGILState_Release, and thread - primitives, also find a case where CPython will allow thread creation - before PyEval_InitThreads is run, dissallow on PyPy - - create a PyObject-specific list strategy - - rewrite slot assignment for typeobjects - - improve tracking of PyObject to rpython object mapping - - support tp_as_{number, sequence, mapping, buffer} slots - -(makes the pypy-c bigger; this was fixed subsequently by the -share-cpyext-cpython-api branch) - -.. branch: share-mapdict-methods-2 - -Reduce generated code for subclasses by using the same function objects in all -generated subclasses. - -.. branch: share-cpyext-cpython-api - -.. branch: cpyext-auto-gil - -CPyExt tweak: instead of "GIL not held when a CPython C extension module -calls PyXxx", we now silently acquire/release the GIL. Helps with -CPython C extension modules that call some PyXxx() functions without -holding the GIL (arguably, they are theorically buggy). - -.. branch: cpyext-test-A - -Get the cpyext tests to pass with "-A" (i.e. when tested directly with -CPython). - -.. branch: oefmt - -.. branch: cpyext-werror - -Compile c snippets with -Werror in cpyext - -.. branch: gc-del-3 - -Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst. -It is a more flexible way to make RPython finalizers. - -.. branch: unpacking-cpython-shortcut - -.. branch: cleanups - -.. branch: cpyext-more-slots - -.. branch: use-gc-del-3 - -Use the new rgc.FinalizerQueue mechanism to clean up the handling of -``__del__`` methods. Fixes notably issue #2287. (All RPython -subclasses of W_Root need to use FinalizerQueue now.) - -.. branch: ufunc-outer - -Implement ufunc.outer on numpypy - -.. branch: z196-support - -Fixes a critical issue in the register allocator and extends support on s390x. PyPy runs and translates on -the s390x revisions z10 (released February 2008, experimental) and z196 (released August 2010) -) in addition to zEC12 and z13. To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your -shell environment. - diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -1,5 +1,5 @@ ========================= -What's new in PyPy 5.1+ +What's new in PyPy2.7 5.3 ========================= .. this is a revision shortly after release-5.1 @@ -94,10 +94,60 @@ Implement ufunc.outer on numpypy +.. branch: verbose-imports + +Support ``pypy -v``: verbose imports. It does not log as much as +cpython, but it should be enough to help when debugging package layout +problems. + +.. branch: cpyext-macros-cast + +Fix some warnings when compiling CPython C extension modules + +.. branch: syntax_fix + +.. branch: remove-raisingops + +Remove most of the _ovf, _zer and _val operations from RPython. Kills +quite some code internally, and allows the JIT to do better +optimizations: for example, app-level code like ``x / 2`` or ``x % 2`` +can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly +negative. + +.. branch: cpyext-old-buffers + +Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap + +.. branch: numpy-includes + +Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy +This allows building upstream numpy and scipy in pypy via cpyext + +.. branch: traceviewer-common-merge-point-formats + +Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats. + +.. branch: cpyext-pickle + +Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch +at cpyext import time + +.. branch: nonmovable-list + +Add a way to ask "give me a raw pointer to this list's +items". Only for resizable lists of primitives. Turns the GcArray +nonmovable, possibly making a copy of it first. + +.. branch: cpyext-ext + +Finish the work already partially merged in cpyext-for-merge. Adds support +for ByteArrayObject using the nonmovable-list, which also enables +buffer(bytearray()) + .. branch: z196-support -Fixes a critical issue in the register allocator and extends support on s390x. PyPy runs and translates on -the s390x revisions z10 (released February 2008, experimental) and z196 (released August 2010) -) in addition to zEC12 and z13. To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your -shell environment. +Fixes a critical issue in the register allocator and extends support on s390x. +PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental) +and z196 (released August 2010) in addition to zEC12 and z13. +To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst @@ -0,0 +1,10 @@ +================================= +What's new in PyPy3 5.1.1 alpha 1 +================================= + +.. A recent revision, ignoring all other branches for this release +.. startrev: 29d14733e007 + +.. branch: py3.3 + +Python 3.3 compatibility diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -238,6 +238,15 @@ for use. The release packaging script will pick up the tcltk runtime in the lib directory and put it in the archive. +The lzma compression library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python 3.3 ship with CFFI wrappers for the lzma library, which can be +downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version +5.0.5, a prebuilt version can be downloaded from +http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature +http://tukaani.org/xz/xz-5.0.5-windows.zip.sig + Using the mingw compiler ------------------------ diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -9,7 +9,7 @@ from rpython.config.config import to_optparse, make_dict, SUPPRESS_USAGE from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace -from pypy.conftest import pypydir +from pypy import pypydir from rpython.rlib import rthread from pypy.module.thread import os_thread @@ -293,7 +293,7 @@ self.hack_for_cffi_modules(driver) return self.get_entry_point(config) - + def hack_for_cffi_modules(self, driver): # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports @@ -320,7 +320,7 @@ while not basedir.join('include').exists(): _basedir = basedir.dirpath() if _basedir == basedir: - raise ValueError('interpreter %s not inside pypy repo', + raise ValueError('interpreter %s not inside pypy repo', str(exename)) basedir = _basedir modules = self.config.objspace.usemodules.getpaths() diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -2,7 +2,7 @@ # This is pure Python code that handles the main entry point into "pypy". # See test/test_app_main. -# Missing vs CPython: -d, -t, -v, -x, -3 +# Missing vs CPython: -d, -t, -x, -3 USAGE1 = __doc__ = """\ Options and arguments (and corresponding environment variables): -B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x @@ -19,6 +19,8 @@ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE -S : don't imply 'import site' on initialization -u : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x +-v : verbose (trace import statements); also PYTHONVERBOSE=x + can be supplied multiple times to increase verbosity -V : print the Python version number and exit (also --version) -W arg : warning control; arg is action:message:category:module:lineno also PYTHONWARNINGS=arg @@ -529,6 +531,7 @@ warnoptions, unbuffered, ignore_environment, + verbose, **ignored): # with PyPy in top of CPython we can only have around 100 # but we need more in the translated PyPy for the compiler package @@ -580,6 +583,12 @@ if hasattr(signal, 'SIGXFSZ'): signal.signal(signal.SIGXFSZ, signal.SIG_IGN) + # Pre-load the default encoder (controlled by PYTHONIOENCODING) now. + # This is needed before someone mucks up with sys.path (or even adds + # a unicode string to it, leading to infinite recursion when we try + # to encode it during importing). Note: very obscure. Issue #2314. + str(u'') + def inspect_requested(): # We get an interactive prompt in one of the following three cases: # @@ -600,6 +609,11 @@ ((inspect or (readenv and real_getenv('PYTHONINSPECT'))) and sys.stdin.isatty())) + try: + from _ast import PyCF_ACCEPT_NULL_BYTES + except ImportError: + PyCF_ACCEPT_NULL_BYTES = 0 + future_flags = [0] success = True try: @@ -610,7 +624,9 @@ @hidden_applevel def run_it(): - exec run_command in mainmodule.__dict__ + co_cmd = compile(run_command, '', 'exec') + exec co_cmd in mainmodule.__dict__ + future_flags[0] = co_cmd.co_flags success = run_toplevel(run_it) elif run_module: # handle the "-m" command @@ -622,11 +638,6 @@ # handle the case where no command/filename/module is specified # on the command-line. - try: - from _ast import PyCF_ACCEPT_NULL_BYTES - except ImportError: - PyCF_ACCEPT_NULL_BYTES = 0 - # update sys.path *after* loading site.py, in case there is a # "site.py" file in the script's directory. Only run this if we're # executing the interactive prompt, if we're running a script we @@ -653,6 +664,7 @@ 'exec', PyCF_ACCEPT_NULL_BYTES) exec co_python_startup in mainmodule.__dict__ + future_flags[0] = co_python_startup.co_flags mainmodule.__file__ = python_startup run_toplevel(run_it) try: @@ -663,11 +675,14 @@ inspect = True else: # If not interactive, just read and execute stdin normally. + if verbose: + print_banner(not no_site) @hidden_applevel def run_it(): co_stdin = compile(sys.stdin.read(), '', 'exec', PyCF_ACCEPT_NULL_BYTES) exec co_stdin in mainmodule.__dict__ + future_flags[0] = co_stdin.co_flags mainmodule.__file__ = '' success = run_toplevel(run_it) else: @@ -697,7 +712,20 @@ args = (runpy._run_module_as_main, '__main__', False) else: # no. That's the normal path, "pypy stuff.py". - args = (execfile, filename, mainmodule.__dict__) + # This includes the logic from execfile(), tweaked + # to grab the future_flags at the end. + @hidden_applevel + def run_it(): + f = file(filename, 'rU') + try: + source = f.read() + finally: + f.close() + co_main = compile(source.rstrip()+"\n", filename, + 'exec', PyCF_ACCEPT_NULL_BYTES) + exec co_main in mainmodule.__dict__ + future_flags[0] = co_main.co_flags + args = (run_it,) success = run_toplevel(*args) except SystemExit as e: @@ -710,12 +738,21 @@ # start a prompt if requested if inspect_requested(): try: + import __future__ from _pypy_interact import interactive_console pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info) irc_topic = pypy_version_info[3] != 'final' or ( readenv and os.getenv('PYPY_IRC_TOPIC')) + flags = 0 + for fname in __future__.all_feature_names: + feature = getattr(__future__, fname) + if future_flags[0] & feature.compiler_flag: + flags |= feature.compiler_flag + kwds = {} + if flags: + kwds['future_flags'] = flags success = run_toplevel(interactive_console, mainmodule, - quiet=not irc_topic) + quiet=not irc_topic, **kwds) except SystemExit as e: status = e.code else: @@ -724,10 +761,10 @@ return status def print_banner(copyright): - print 'Python %s on %s' % (sys.version, sys.platform) + print >> sys.stderr, 'Python %s on %s' % (sys.version, sys.platform) if copyright: - print ('Type "help", "copyright", "credits" or ' - '"license" for more information.') + print >> sys.stderr, ('Type "help", "copyright", "credits" or ' + '"license" for more information.') STDLIB_WARNING = """\ debug: WARNING: Library path not found, using compiled-in sys.path. diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -564,7 +564,6 @@ self.emit_jump(ops.JUMP_FORWARD, end) self.use_next_block(next_except) self.emit_op(ops.END_FINALLY) # this END_FINALLY will always re-raise - self.is_dead_code() self.use_next_block(otherwise) self.visit_sequence(te.orelse) self.use_next_block(end) diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py --- a/pypy/interpreter/astcompiler/test/test_ast.py +++ b/pypy/interpreter/astcompiler/test/test_ast.py @@ -1,8 +1,8 @@ from pypy.interpreter.astcompiler import ast class TestAstToObject: def test_types(self, space): - assert space.is_true(space.issubtype( - ast.get(space).w_Module, ast.get(space).w_mod)) + assert space.issubtype_w( + ast.get(space).w_Module, ast.get(space).w_mod) def test_num(self, space): value = space.wrap(42) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -243,6 +243,9 @@ def unicode_w(self, space): self._typed_unwrap_error(space, "unicode") + def bytearray_list_of_chars_w(self, space): + self._typed_unwrap_error(space, "bytearray") + def int_w(self, space, allow_conversion=True): # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w # raises w_TypeError @@ -1215,7 +1218,7 @@ def abstract_issubclass_w(self, w_cls1, w_cls2): # Equivalent to 'issubclass(cls1, cls2)'. - return self.is_true(self.issubtype(w_cls1, w_cls2)) + return self.issubtype_w(w_cls1, w_cls2) def abstract_isinstance_w(self, w_obj, w_cls): # Equivalent to 'isinstance(obj, cls)'. @@ -1237,16 +1240,16 @@ def exception_is_valid_obj_as_class_w(self, w_obj): if not self.isinstance_w(w_obj, self.w_type): return False - return self.is_true(self.issubtype(w_obj, self.w_BaseException)) + return self.issubtype_w(w_obj, self.w_BaseException) def exception_is_valid_class_w(self, w_cls): - return self.is_true(self.issubtype(w_cls, self.w_BaseException)) + return self.issubtype_w(w_cls, self.w_BaseException) def exception_getclass(self, w_obj): return self.type(w_obj) def exception_issubclass_w(self, w_cls1, w_cls2): - return self.is_true(self.issubtype(w_cls1, w_cls2)) + return self.issubtype_w(w_cls1, w_cls2) def new_exception_class(self, *args, **kwargs): "NOT_RPYTHON; convenience method to create excceptions in modules" diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -6,7 +6,7 @@ import sys, os, re, runpy, subprocess from rpython.tool.udir import udir from contextlib import contextmanager -from pypy.conftest import pypydir +from pypy import pypydir from lib_pypy._pypy_interact import irc_header try: @@ -76,6 +76,11 @@ print 'Goodbye2' # should not be reached """) +script_with_future = getscript(""" + from __future__ import division + from __future__ import print_function + """) + class TestParseCommandLine: def check_options(self, options, sys_argv, **expected): @@ -286,7 +291,7 @@ child.expect('>>>') # banner if irc_topic: assert irc_header in child.before - else: + else: assert irc_header not in child.before def test_help(self): @@ -445,6 +450,31 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_future_in_executed_script(self): + child = self.spawn(['-i', script_with_future]) + child.expect('>>> ') + child.sendline('x=1; print(x/2, 3/4)') + child.expect('0.5 0.75') + + def test_future_in_python_startup(self, monkeypatch): + monkeypatch.setenv('PYTHONSTARTUP', script_with_future) + child = self.spawn([]) + child.expect('>>> ') + child.sendline('x=1; print(x/2, 3/4)') + child.expect('0.5 0.75') + + def test_future_in_cmd(self): + child = self.spawn(['-i', '-c', 'from __future__ import division']) + child.expect('>>> ') + child.sendline('x=1; x/2; 3/4') + child.expect('0.5') + child.expect('0.75') + + def test_cmd_co_name(self): + child = self.spawn(['-c', + 'import sys; print sys._getframe(0).f_code.co_name']) + child.expect('') + def test_ignore_python_inspect(self): os.environ['PYTHONINSPECT_'] = '1' try: @@ -1044,4 +1074,4 @@ # assert it did not crash finally: sys.path[:] = old_sys_path - + diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -48,10 +48,10 @@ return f.f_code assert g() is g.func_code - def test_f_trace_del(self): + def test_f_trace_del(self): import sys - f = sys._getframe() - del f.f_trace + f = sys._getframe() + del f.f_trace assert f.f_trace is None def test_f_lineno(self): @@ -116,7 +116,7 @@ def f(): assert sys._getframe().f_code.co_name == g() def g(): - return sys._getframe().f_back.f_code.co_name + return sys._getframe().f_back.f_code.co_name f() def test_f_back_virtualref(self): @@ -233,7 +233,7 @@ def test_trace_exc(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -298,7 +298,7 @@ def test_trace_return_exc(self): import sys l = [] - def trace(a,b,c): + def trace(a,b,c): if b in ('exception', 'return'): l.append((b, c)) return trace @@ -444,7 +444,7 @@ def test_dont_trace_on_reraise(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace @@ -466,7 +466,7 @@ def test_dont_trace_on_raise_with_tb(self): import sys l = [] - def ltrace(a,b,c): + def ltrace(a,b,c): if b == 'exception': l.append(c) return ltrace diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -3,7 +3,7 @@ from pypy.interpreter import gateway from rpython.rlib.jit import non_virtual_ref, vref_None -class AppTestSlow: +class AppTestSlow: spaceconfig = dict(usemodules=['itertools']) def setup_class(cls): @@ -64,7 +64,7 @@ space.setitem(space.builtin.w_dict, space.wrap('read_exc_type'), space.wrap(read_exc_type_gw)) - + def _detach_helpers(space): space.delitem(space.builtin.w_dict, space.wrap('hide_top_frame')) @@ -92,7 +92,7 @@ pckl = pickle.dumps(code) result = pickle.loads(pckl) assert code == result - + def test_pickle_global_func(self): import new mod = new.module('mod') @@ -109,7 +109,7 @@ assert func is result finally: del sys.modules['mod'] - + def test_pickle_not_imported_module(self): import new mod = new.module('mod') @@ -119,13 +119,13 @@ result = pickle.loads(pckl) assert mod.__name__ == result.__name__ assert mod.__dict__ == result.__dict__ - + def test_pickle_builtin_func(self): import pickle pckl = pickle.dumps(map) result = pickle.loads(pckl) assert map is result - + def test_pickle_non_top_reachable_func(self): def func(): return 42 @@ -142,7 +142,7 @@ assert func.func_dict == result.func_dict assert func.func_doc == result.func_doc assert func.func_globals == result.func_globals - + def test_pickle_cell(self): def g(): x = [42] @@ -171,7 +171,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) assert type(f1) is type(f2) @@ -223,7 +223,7 @@ f1 = f() saved = hide_top_frame(f1) pckl = pickle.dumps(f1) - restore_top_frame(f1, saved) + restore_top_frame(f1, saved) f2 = pickle.loads(pckl) def test_frame_setstate_crash(self): @@ -257,21 +257,21 @@ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_moduledict(self): import pickle moddict = pickle.__dict__ pckl = pickle.dumps(moddict) result = pickle.loads(pckl) assert moddict is result - + def test_pickle_bltins_module(self): import pickle mod = __builtins__ pckl = pickle.dumps(mod) result = pickle.loads(pckl) assert mod is result - + def test_pickle_buffer(self): skip("Can't pickle buffer objects on top of CPython either. " "Do we really need it?") @@ -280,14 +280,14 @@ pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_complex(self): import pickle a = complex(1.23,4.567) pckl = pickle.dumps(a) result = pickle.loads(pckl) assert a == result - + def test_pickle_method(self): class myclass(object): def f(self): @@ -308,7 +308,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_staticmethod(self): class myclass(object): def f(): @@ -319,7 +319,7 @@ pckl = pickle.dumps(method) result = pickle.loads(pckl) assert method() == result() - + def test_pickle_classmethod(self): class myclass(object): def f(cls): @@ -337,7 +337,7 @@ assert method() == result() finally: del sys.modules['mod'] - + def test_pickle_sequenceiter(self): ''' In PyPy there is no distinction here between listiterator and diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -12,7 +12,8 @@ class TypeDef(object): - def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict): + def __init__(self, __name, __base=None, __total_ordering__=None, + __buffer=None, **rawdict): "NOT_RPYTHON: initialization-time only" self.name = __name if __base is None: @@ -22,6 +23,9 @@ else: bases = [__base] self.bases = bases + # Used in cpyext to fill tp_as_buffer slots + assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" + self.buffer = __buffer self.heaptype = False self.hasdict = '__dict__' in rawdict # no __del__: use an RPython _finalize_() method and register_finalizer diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -86,8 +86,8 @@ 'max' : 'functional.max', 'reversed' : 'functional.reversed', 'super' : 'descriptor.W_Super', - 'staticmethod' : 'descriptor.StaticMethod', - 'classmethod' : 'descriptor.ClassMethod', + 'staticmethod' : 'pypy.interpreter.function.StaticMethod', + 'classmethod' : 'pypy.interpreter.function.ClassMethod', 'property' : 'descriptor.W_Property', 'globals' : 'interp_inspect.globals', diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py --- a/pypy/module/__builtin__/abstractinst.py +++ b/pypy/module/__builtin__/abstractinst.py @@ -76,11 +76,10 @@ w_pretendtype = space.getattr(w_obj, space.wrap('__class__')) if space.is_w(w_pretendtype, space.type(w_obj)): return False # common case: obj.__class__ is type(obj) - if allow_override: - w_result = space.issubtype_allow_override(w_pretendtype, - w_klass_or_tuple) - else: - w_result = space.issubtype(w_pretendtype, w_klass_or_tuple) + if not allow_override: + return space.issubtype_w(w_pretendtype, w_klass_or_tuple) + w_result = space.issubtype_allow_override(w_pretendtype, + w_klass_or_tuple) except OperationError as e: if e.async(space): raise @@ -137,11 +136,9 @@ # -- case (type, type) try: - if allow_override: - w_result = space.issubtype_allow_override(w_derived, - w_klass_or_tuple) - else: - w_result = space.issubtype(w_derived, w_klass_or_tuple) + if not allow_override: + return space.issubtype_w(w_derived, w_klass_or_tuple) + w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple) except OperationError as e: # if one of the args was not a type, ignore it if not e.match(space, space.w_TypeError): raise # propagate other errors diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py --- a/pypy/module/__builtin__/descriptor.py +++ b/pypy/module/__builtin__/descriptor.py @@ -1,31 +1,39 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.function import StaticMethod, ClassMethod -from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from pypy.interpreter.typedef import (TypeDef, interp_attrproperty_w, - generic_new_descr) +from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.interpreter.typedef import ( + TypeDef, generic_new_descr, interp_attrproperty_w) from pypy.objspace.descroperation import object_getattribute class W_Super(W_Root): - def __init__(self, space, w_starttype, w_objtype, w_self): + + def __init__(self, space): + self.w_starttype = None + self.w_objtype = None + self.w_self = None + + def descr_init(self, space, w_starttype, w_obj_or_type=None): + if space.is_none(w_obj_or_type): + w_type = None # unbound super object + w_obj_or_type = space.w_None + else: + w_type = _super_check(space, w_starttype, w_obj_or_type) self.w_starttype = w_starttype - self.w_objtype = w_objtype - self.w_self = w_self + self.w_objtype = w_type + self.w_self = w_obj_or_type def get(self, space, w_obj, w_type=None): - w = space.wrap if self.w_self is None or space.is_w(w_obj, space.w_None): - return w(self) + return self else: # if type(self) is W_Super: # XXX write a fast path for this common case - w_selftype = space.type(w(self)) + w_selftype = space.type(self) return space.call_function(w_selftype, self.w_starttype, w_obj) - @unwrap_spec(name=str) - def getattribute(self, space, name): - w = space.wrap + def getattribute(self, space, w_name): + name = space.str_w(w_name) # only use a special logic for bound super objects and not for # getting the __class__ of the super object itself. if self.w_objtype is not None and name != '__class__': @@ -45,44 +53,42 @@ return space.get_and_call_function(w_get, w_value, w_obj, self.w_objtype) # fallback to object.__getattribute__() - return space.call_function(object_getattribute(space), - w(self), w(name)) + return space.call_function(object_getattribute(space), self, w_name) -def descr_new_super(space, w_subtype, w_starttype, w_obj_or_type=None): - if space.is_none(w_obj_or_type): - w_type = None # unbound super object - w_obj_or_type = space.w_None - else: - w_objtype = space.type(w_obj_or_type) - if space.is_true(space.issubtype(w_objtype, space.w_type)) and \ - space.is_true(space.issubtype(w_obj_or_type, w_starttype)): - w_type = w_obj_or_type # special case for class methods - elif space.is_true(space.issubtype(w_objtype, w_starttype)): - w_type = w_objtype # normal case - else: - try: - w_type = space.getattr(w_obj_or_type, space.wrap('__class__')) - except OperationError as o: - if not o.match(space, space.w_AttributeError): - raise - w_type = w_objtype - if not space.is_true(space.issubtype(w_type, w_starttype)): - raise oefmt(space.w_TypeError, - "super(type, obj): obj must be an instance or " - "subtype of type") - # XXX the details of how allocate_instance() should be used are not - # really well defined - w_result = space.allocate_instance(W_Super, w_subtype) From pypy.commits at gmail.com Tue Jun 7 08:26:59 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 05:26:59 -0700 (PDT) Subject: [pypy-commit] pypy z196-support: close branch Message-ID: <5756bd93.2450c20a.b8f3a.fffff878@mx.google.com> Author: Richard Plangger Branch: z196-support Changeset: r84991:0ce4800ba09c Date: 2016-06-07 14:25 +0200 http://bitbucket.org/pypy/pypy/changeset/0ce4800ba09c/ Log: close branch From pypy.commits at gmail.com Tue Jun 7 08:27:01 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 05:27:01 -0700 (PDT) Subject: [pypy-commit] pypy default: merged z196-support Message-ID: <5756bd95.07a6c20a.9c49c.ffffc449@mx.google.com> Author: Richard Plangger Branch: Changeset: r84992:8b000195dd94 Date: 2016-06-07 14:26 +0200 http://bitbucket.org/pypy/pypy/changeset/8b000195dd94/ Log: merged z196-support diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-pypy2-5.3.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -143,3 +143,11 @@ Finish the work already partially merged in cpyext-for-merge. Adds support for ByteArrayObject using the nonmovable-list, which also enables buffer(bytearray()) + +.. branch: z196-support + +Fixes a critical issue in the register allocator and extends support on s390x. +PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental) +and z196 (released August 2010) in addition to zEC12 and z13. +To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. + diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py --- a/pypy/module/select/test/test_epoll.py +++ b/pypy/module/select/test/test_epoll.py @@ -20,6 +20,10 @@ self.w_sockets = self.space.wrap([]) if platform.machine().startswith('arm'): self.w_timeout = self.space.wrap(0.06) + if platform.machine().startswith('s390x'): + # s390x is not slow, but it seems there is one case when epoll + # modify method is called that takes longer on s390x + self.w_timeout = self.space.wrap(0.06) else: self.w_timeout = self.space.wrap(0.02) diff --git a/rpython/doc/arch/index.rst b/rpython/doc/arch/index.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/arch/index.rst @@ -0,0 +1,11 @@ +.. _arch_index: + +Architecture specific notes +=========================== + +Here you can find some architecture specific notes. + +.. toctree:: + :maxdepth: 1 + + s390x diff --git a/rpython/doc/arch/s390x.rst b/rpython/doc/arch/s390x.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/arch/s390x.rst @@ -0,0 +1,34 @@ +.. _s390x: + +IBM Mainframe S390X +=================== + +Our JIT implements the 64 bit version of the IBM Mainframe called s390x. +Note that this architecture is big endian. + +Currently supported ISAs: + +* z13 (released January 2015) +* zEC12 (released September 2012) +* z196 (released August 2010) +* z10 (released February 2008) + +To check if all the necessary CPU facilities are installed +on the subject machine, please run the test using a copy of the pypy +source code:: + + $ ./pytest.py rpython/jit/backend/zarch/test/test_assembler -v -k 'test_facility' + +In addition you can run the auto encoding test to check if your Linux GCC tool chain +is able to compile all instructions used in the JIT backend:: + + $ ./pytest.py rpython/jit/backend/zarch/test/test_auto_encoding.py -v + +Translating +----------- + +Specifically check for these two dependencies. On old versions of some +Linux distributions ship older versions. + +* libffi (version should do > 3.0.+). +* CPython 2.7.+. diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -37,7 +37,6 @@ arm logging - s390x Writing your own interpreter in RPython @@ -61,6 +60,7 @@ getting-started dir-reference jit/index + arch/index translation rtyper garbage_collection diff --git a/rpython/doc/s390x.rst b/rpython/doc/s390x.rst deleted file mode 100644 --- a/rpython/doc/s390x.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _s390x: - -S390X JIT Backend -================= - -Our JIT implements the 64 bit version of the IBM Mainframe called s390x. -Note that this architecture is big endian. - -The following facilities need to be installed to operate -correctly (all of the machines used for development these where installed): - -* General-Instructions-Extension -* Long-Displacement -* Binary Floating Point (IEEE) - -Translating ------------ - -Ensure that libffi is installed (version should do > 3.0.+). -CPython should be version 2.7.+. diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -44,7 +44,6 @@ # rotating 'RISBG': ('rie_f', ['\xEC','\x55']), - 'RISBGN': ('rie_f', ['\xEC','\x59']), # invert & negative & absolute 'LPGR': ('rre', ['\xB9','\x00']), diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py --- a/rpython/jit/backend/zarch/test/test_assembler.py +++ b/rpython/jit/backend/zarch/test/test_assembler.py @@ -155,7 +155,15 @@ s64 = bin(fac_data[1])[2:] print(f64) print(s64) + for i,c in enumerate(f64): + print('index: %d is set? %s' % (i,c)) + + assert f64[1] == '1' # The z/Architecture architectural mode is installed. + assert f64[2] == '1' # The z/Architecture architectural mode is active. assert f64[18] == '1' # long displacement facility + assert f64[21] == '1' # extended immediate facility + assert f64[34] == '1' # general instruction facility + assert f64[41] == '1' # floating-point-support-enhancement def test_load_byte_zero_extend(self): adr = self.a.datablockwrapper.malloc_aligned(16, 16) @@ -189,7 +197,7 @@ @py.test.mark.parametrize('p', [2**32,2**32+1,2**63-1,2**63-2,0,1,2,3,4,5,6,7,8,10001]) def test_align_withroll(self, p): self.a.mc.load_imm(r.r2, p & 0xffffFFFFffffFFFF) - self.a.mc.RISBGN(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) + self.a.mc.RISBG(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) self.a.mc.BCR(con.ANY, r.r14) assert run_asm(self.a) == rffi.cast(rffi.ULONG,p) & ~(7) @@ -214,7 +222,7 @@ n = 13 l = loc self.a.mc.load_imm(r.r2, 7< Author: Richard Plangger Branch: s390x-5.3-catchup Changeset: r84993:772d2e282f99 Date: 2016-06-07 14:27 +0200 http://bitbucket.org/pypy/pypy/changeset/772d2e282f99/ Log: merged default diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-pypy2-5.3.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -143,3 +143,11 @@ Finish the work already partially merged in cpyext-for-merge. Adds support for ByteArrayObject using the nonmovable-list, which also enables buffer(bytearray()) + +.. branch: z196-support + +Fixes a critical issue in the register allocator and extends support on s390x. +PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental) +and z196 (released August 2010) in addition to zEC12 and z13. +To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. + diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py --- a/pypy/module/select/test/test_epoll.py +++ b/pypy/module/select/test/test_epoll.py @@ -20,6 +20,10 @@ self.w_sockets = self.space.wrap([]) if platform.machine().startswith('arm'): self.w_timeout = self.space.wrap(0.06) + if platform.machine().startswith('s390x'): + # s390x is not slow, but it seems there is one case when epoll + # modify method is called that takes longer on s390x + self.w_timeout = self.space.wrap(0.06) else: self.w_timeout = self.space.wrap(0.02) diff --git a/rpython/doc/arch/index.rst b/rpython/doc/arch/index.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/arch/index.rst @@ -0,0 +1,11 @@ +.. _arch_index: + +Architecture specific notes +=========================== + +Here you can find some architecture specific notes. + +.. toctree:: + :maxdepth: 1 + + s390x diff --git a/rpython/doc/arch/s390x.rst b/rpython/doc/arch/s390x.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/arch/s390x.rst @@ -0,0 +1,34 @@ +.. _s390x: + +IBM Mainframe S390X +=================== + +Our JIT implements the 64 bit version of the IBM Mainframe called s390x. +Note that this architecture is big endian. + +Currently supported ISAs: + +* z13 (released January 2015) +* zEC12 (released September 2012) +* z196 (released August 2010) +* z10 (released February 2008) + +To check if all the necessary CPU facilities are installed +on the subject machine, please run the test using a copy of the pypy +source code:: + + $ ./pytest.py rpython/jit/backend/zarch/test/test_assembler -v -k 'test_facility' + +In addition you can run the auto encoding test to check if your Linux GCC tool chain +is able to compile all instructions used in the JIT backend:: + + $ ./pytest.py rpython/jit/backend/zarch/test/test_auto_encoding.py -v + +Translating +----------- + +Specifically check for these two dependencies. On old versions of some +Linux distributions ship older versions. + +* libffi (version should do > 3.0.+). +* CPython 2.7.+. diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -37,7 +37,6 @@ arm logging - s390x Writing your own interpreter in RPython @@ -61,6 +60,7 @@ getting-started dir-reference jit/index + arch/index translation rtyper garbage_collection diff --git a/rpython/doc/s390x.rst b/rpython/doc/s390x.rst deleted file mode 100644 --- a/rpython/doc/s390x.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _s390x: - -S390X JIT Backend -================= - -Our JIT implements the 64 bit version of the IBM Mainframe called s390x. -Note that this architecture is big endian. - -The following facilities need to be installed to operate -correctly (all of the machines used for development these where installed): - -* General-Instructions-Extension -* Long-Displacement -* Binary Floating Point (IEEE) - -Translating ------------ - -Ensure that libffi is installed (version should do > 3.0.+). -CPython should be version 2.7.+. diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -45,7 +45,6 @@ # rotating 'RISBG': ('rie_f', ['\xEC','\x55']), - 'RISBGN': ('rie_f', ['\xEC','\x59']), # invert & negative & absolute 'LPGR': ('rre', ['\xB9','\x00']), diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py --- a/rpython/jit/backend/zarch/test/test_assembler.py +++ b/rpython/jit/backend/zarch/test/test_assembler.py @@ -155,7 +155,15 @@ s64 = bin(fac_data[1])[2:] print(f64) print(s64) + for i,c in enumerate(f64): + print('index: %d is set? %s' % (i,c)) + + assert f64[1] == '1' # The z/Architecture architectural mode is installed. + assert f64[2] == '1' # The z/Architecture architectural mode is active. assert f64[18] == '1' # long displacement facility + assert f64[21] == '1' # extended immediate facility + assert f64[34] == '1' # general instruction facility + assert f64[41] == '1' # floating-point-support-enhancement def test_load_byte_zero_extend(self): adr = self.a.datablockwrapper.malloc_aligned(16, 16) @@ -189,7 +197,7 @@ @py.test.mark.parametrize('p', [2**32,2**32+1,2**63-1,2**63-2,0,1,2,3,4,5,6,7,8,10001]) def test_align_withroll(self, p): self.a.mc.load_imm(r.r2, p & 0xffffFFFFffffFFFF) - self.a.mc.RISBGN(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) + self.a.mc.RISBG(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0)) self.a.mc.BCR(con.ANY, r.r14) assert run_asm(self.a) == rffi.cast(rffi.ULONG,p) & ~(7) @@ -214,7 +222,7 @@ n = 13 l = loc self.a.mc.load_imm(r.r2, 7< Author: Richard Plangger Branch: s390x-5.3-catchup Changeset: r84994:91020aa67d04 Date: 2016-06-07 15:03 +0200 http://bitbucket.org/pypy/pypy/changeset/91020aa67d04/ Log: close branch From pypy.commits at gmail.com Tue Jun 7 09:06:13 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:06:13 -0700 (PDT) Subject: [pypy-commit] pypy default: (s390x) merged s390x-5.3-catchup Message-ID: <5756c6c5.2946c20a.8485e.0427@mx.google.com> Author: Richard Plangger Branch: Changeset: r84995:02f61acb357d Date: 2016-06-07 15:04 +0200 http://bitbucket.org/pypy/pypy/changeset/02f61acb357d/ Log: (s390x) merged s390x-5.3-catchup diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -439,7 +439,7 @@ prepare_int_lshift = helper.prepare_binary_op prepare_int_rshift = helper.prepare_binary_op prepare_uint_rshift = helper.prepare_binary_op - prepare_uint_mul_high = helper.prepare_binary_op + prepare_uint_mul_high = helper.prepare_int_mul_ovf prepare_int_add_ovf = helper.prepare_binary_op prepare_int_sub_ovf = helper.prepare_binary_op diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -29,6 +29,7 @@ 'MGHI': ('ri', ['\xA7','\x0D']), 'MSGFI': ('ril', ['\xC2','\x00']), 'MLGR': ('rre', ['\xB9','\x86'], 'eo,r'), + 'MLG': ('rxy', ['\xE3','\x86'], 'eo,bid'), # div/mod 'DSGR': ('rre', ['\xB9','\x0D'], 'eo,r'), 'DSG': ('rxy', ['\xE3','\x0D'], 'eo,bidl'), diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -160,11 +160,15 @@ omc.BRC(c.ANY, l.imm(label_end - jmp_neither_lqlr_overflow)) omc.overwrite() - emit_int_floordiv = gen_emit_div_mod('DSGR', 'DSG') - emit_uint_floordiv = gen_emit_div_mod('DLGR', 'DLG') - # NOTE division sets one register with the modulo value, thus - # the regalloc ensures the right register survives. - emit_int_mod = gen_emit_div_mod('DSGR', 'DSG') + def emit_uint_mul_high(self, op, arglocs, regalloc): + r0, _, a1 = arglocs + # _ carries the value, contents of r0 are ignored + assert not r0.is_imm() + assert not a1.is_imm() + if a1.is_core_reg(): + self.mc.MLGR(r0, a1) + else: + self.mc.MLG(r0, a1) def emit_int_invert(self, op, arglocs, regalloc): l0, = arglocs diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -733,9 +733,6 @@ prepare_int_sub_ovf = helper.prepare_int_sub prepare_int_mul = helper.prepare_int_mul prepare_int_mul_ovf = helper.prepare_int_mul_ovf - prepare_int_floordiv = helper.prepare_int_div - prepare_uint_floordiv = helper.prepare_int_div - prepare_int_mod = helper.prepare_int_mod prepare_nursery_ptr_increment = prepare_int_add prepare_int_and = helper.prepare_int_logic @@ -746,6 +743,18 @@ prepare_int_lshift = helper.prepare_int_shift prepare_uint_rshift = helper.prepare_int_shift + def prepare_uint_mul_high(self, op): + a0 = op.getarg(0) + a1 = op.getarg(1) + if a0.is_constant(): + a0, a1 = a1, a0 + if helper.check_imm32(a1): + l1 = self.ensure_reg(a1) + else: + l1 = self.ensure_reg_or_pool(a1) + lr,lq = self.rm.ensure_even_odd_pair(a0, op, bind_first=True) + return [lr, lq, l1] + prepare_int_le = helper.generate_cmp_op() prepare_int_lt = helper.generate_cmp_op() prepare_int_ge = helper.generate_cmp_op() diff --git a/rpython/jit/backend/zarch/test/test_int.py b/rpython/jit/backend/zarch/test/test_int.py --- a/rpython/jit/backend/zarch/test/test_int.py +++ b/rpython/jit/backend/zarch/test/test_int.py @@ -35,41 +35,13 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail == finishdescr # ensures that guard is not taken! - def test_double_evenodd_pair(self): - code = """ - [i0] - i1 = int_floordiv(i0, 2) - i2 = int_floordiv(i0, 3) - i3 = int_floordiv(i0, 4) - i4 = int_floordiv(i0, 5) - i5 = int_floordiv(i0, 6) - i6 = int_floordiv(i0, 7) - i7 = int_floordiv(i0, 8) - i8 = int_le(i1, 0) - guard_true(i8) [i1,i2,i3,i4,i5,i6,i7] - finish(i0, descr=faildescr) - """ - # the guard forces 3 spills because after 4 divisions - # all even slots of the managed registers are full - loop = parse(code, namespace={'faildescr': BasicFinalDescr(1)}) - looptoken = JitCellToken() - self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - deadframe = self.cpu.execute_token(looptoken, 100) - fail = self.cpu.get_latest_descr(deadframe) - for i in range(2,9): - assert self.cpu.get_int_value(deadframe, i-2) == 100//i - - - @py.test.mark.parametrize('value', [2,3,15,2**16]) def test_evenodd_pair_extensive(self, value): instrs = [] failargs = [] values = [] j = 0 - mapping = (('int_floordiv',lambda x,y: x // y), - ('int_mod', lambda x,y: x % y), - ('int_mul_ovf', lambda x,y: x * y)) + mapping = (('int_mul_ovf', lambda x,y: x * y),) for i in range(20): name, func = mapping[j] instrs.append("i{d} = {i}(i0, {d})".format(d=i+1, i=name)) diff --git a/rpython/jit/backend/zarch/test/test_regalloc.py b/rpython/jit/backend/zarch/test/test_regalloc.py --- a/rpython/jit/backend/zarch/test/test_regalloc.py +++ b/rpython/jit/backend/zarch/test/test_regalloc.py @@ -146,128 +146,3 @@ assert cpu.get_int_value(deadframe, 0) == 0 assert cpu.get_int_value(deadframe, 1) == -1000 -def test_bug_0(): - cpu, deadframe = run([-13, 10, 10, 8, -8, -16, -18, 46, -12, 26], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_gt(i3, -48) - i12 = int_xor(i8, i1) - i13 = int_gt(i6, -9) - i14 = int_le(i13, i2) - i15 = int_le(i11, i5) - i16 = uint_ge(i13, i13) - i17 = int_or(i9, -23) - i18 = int_lt(i10, i13) - i19 = int_or(i15, i5) - i20 = int_xor(i17, 54) - i21 = int_mul(i8, i10) - i22 = int_or(i3, i9) - i41 = int_and(i11, -4) - i42 = int_or(i41, 1) - i23 = int_mod(i12, i42) - i24 = int_is_true(i6) - i25 = uint_rshift(i15, 6) - i26 = int_or(-4, i25) - i27 = int_invert(i8) - i28 = int_sub(-113, i11) - i29 = int_neg(i7) - i30 = int_neg(i24) - i31 = int_floordiv(i3, 53) - i32 = int_mul(i28, i27) - i43 = int_and(i18, -4) - i44 = int_or(i43, 1) - i33 = int_mod(i26, i44) - i34 = int_or(i27, i19) - i35 = uint_lt(i13, 1) - i45 = int_and(i21, 31) - i36 = int_rshift(i21, i45) - i46 = int_and(i20, 31) - i37 = uint_rshift(i4, i46) - i38 = uint_gt(i33, -11) - i39 = int_neg(i7) - i40 = int_gt(i24, i32) - i99 = same_as_i(0) - guard_true(i99) [i40, i36, i37, i31, i16, i34, i35, i23, i22, i29, i14, i39, i30, i38] - finish(42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 0 - assert cpu.get_int_value(deadframe, 2) == 0 - assert cpu.get_int_value(deadframe, 3) == 0 - assert cpu.get_int_value(deadframe, 4) == 1 - assert cpu.get_int_value(deadframe, 5) == -7 - assert cpu.get_int_value(deadframe, 6) == 1 - assert cpu.get_int_value(deadframe, 7) == 0 - assert cpu.get_int_value(deadframe, 8) == -2 - assert cpu.get_int_value(deadframe, 9) == 18 - assert cpu.get_int_value(deadframe, 10) == 1 - assert cpu.get_int_value(deadframe, 11) == 18 - assert cpu.get_int_value(deadframe, 12) == -1 - assert cpu.get_int_value(deadframe, 13) == 0 - -def test_bug_1(): - cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_lt(i6, 0) - i41 = int_and(i3, 31) - i12 = int_rshift(i3, i41) - i13 = int_neg(i2) - i14 = int_add(i11, i7) - i15 = int_or(i3, i2) - i16 = int_or(i12, i12) - i17 = int_ne(i2, i5) - i42 = int_and(i5, 31) - i18 = uint_rshift(i14, i42) - i43 = int_and(i14, 31) - i19 = int_lshift(7, i43) - i20 = int_neg(i19) - i21 = int_mod(i3, 1) - i22 = uint_ge(i15, i1) - i44 = int_and(i16, 31) - i23 = int_lshift(i8, i44) - i24 = int_is_true(i17) - i45 = int_and(i5, 31) - i25 = int_lshift(i14, i45) - i26 = int_lshift(i5, 17) - i27 = int_eq(i9, i15) - i28 = int_ge(0, i6) - i29 = int_neg(i15) - i30 = int_neg(i22) - i31 = int_add(i7, i16) - i32 = uint_lt(i19, i19) - i33 = int_add(i2, 1) - i34 = int_neg(i5) - i35 = int_add(i17, i24) - i36 = uint_lt(2, i16) - i37 = int_neg(i9) - i38 = int_gt(i4, i11) - i39 = int_lt(i27, i22) - i40 = int_neg(i27) - i99 = same_as_i(0) - guard_true(i99) [i40, i10, i36, i26, i13, i30, i21, i33, i18, i25, i31, i32, i28, i29, i35, i38, i20, i39, i34, i23, i37] - finish(-42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 8 - assert cpu.get_int_value(deadframe, 2) == 1 - assert cpu.get_int_value(deadframe, 3) == 131072 - assert cpu.get_int_value(deadframe, 4) == 20 - assert cpu.get_int_value(deadframe, 5) == -1 - assert cpu.get_int_value(deadframe, 6) == 0 - assert cpu.get_int_value(deadframe, 7) == -19 - assert cpu.get_int_value(deadframe, 8) == 6 - assert cpu.get_int_value(deadframe, 9) == 26 - assert cpu.get_int_value(deadframe, 10) == 12 - assert cpu.get_int_value(deadframe, 11) == 0 - assert cpu.get_int_value(deadframe, 12) == 0 - assert cpu.get_int_value(deadframe, 13) == 2 - assert cpu.get_int_value(deadframe, 14) == 2 - assert cpu.get_int_value(deadframe, 15) == 1 - assert cpu.get_int_value(deadframe, 16) == -57344 - assert cpu.get_int_value(deadframe, 17) == 1 - assert cpu.get_int_value(deadframe, 18) == -1 - if WORD == 4: - assert cpu.get_int_value(deadframe, 19) == -2147483648 - elif WORD == 8: - assert cpu.get_int_value(deadframe, 19) == 19327352832 - assert cpu.get_int_value(deadframe, 20) == -49 - From pypy.commits at gmail.com Tue Jun 7 09:06:15 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:06:15 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5756c6c7.430ec20a.18677.ffffaf39@mx.google.com> Author: Richard Plangger Branch: Changeset: r84996:f80cf7e37cf9 Date: 2016-06-07 15:05 +0200 http://bitbucket.org/pypy/pypy/changeset/f80cf7e37cf9/ Log: document merged branch diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst --- a/pypy/doc/whatsnew-pypy2-5.3.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst @@ -151,3 +151,6 @@ and z196 (released August 2010) in addition to zEC12 and z13. To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. +.. branch: s390x-5.3-catchup + +Implement the backend related changes for s390x. From pypy.commits at gmail.com Tue Jun 7 09:11:18 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:11:18 -0700 (PDT) Subject: [pypy-commit] pypy fix-gen-dfa: close branch Message-ID: <5756c7f6.c728c20a.980fe.6854@mx.google.com> Author: Richard Plangger Branch: fix-gen-dfa Changeset: r84998:ca83cb5287e7 Date: 2016-06-07 15:10 +0200 http://bitbucket.org/pypy/pypy/changeset/ca83cb5287e7/ Log: close branch From pypy.commits at gmail.com Tue Jun 7 09:11:16 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:11:16 -0700 (PDT) Subject: [pypy-commit] pypy default: merge branch and document it Message-ID: <5756c7f4.49921c0a.abbd0.ffff91aa@mx.google.com> Author: Richard Plangger Branch: Changeset: r84997:81f0cfc836dc Date: 2016-06-07 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/81f0cfc836dc/ Log: merge branch and document it diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,6 @@ .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 +.. branch: fix-gen-dfa + +Resolves an issue with the generator script to build the dfa for Python syntax. diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -191,7 +191,7 @@ newArcPair(states, EMPTY), pseudoExtras, number, funny, contStr, name)) dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken) - return DFA(dfaStates, dfaAccepts) + return DFA(dfaStates, dfaAccepts), dfaStates # ______________________________________________________________________ @@ -205,7 +205,9 @@ newArcPair(states, DEFAULT), any(states, notGroupStr(states, "'\\")))), newArcPair(states, "'")) - singleDFA = DFA(*nfaToDfa(states, *single)) + states, accepts = nfaToDfa(states, *single) + singleDFA = DFA(states, accepts) + states_singleDFA = states states = [] double = chain(states, any(states, notGroupStr(states, '"\\')), @@ -215,7 +217,9 @@ newArcPair(states, DEFAULT), any(states, notGroupStr(states, '"\\')))), newArcPair(states, '"')) - doubleDFA = DFA(*nfaToDfa(states, *double)) + states, accepts = nfaToDfa(states, *double) + doubleDFA = DFA(states, accepts) + states_doubleDFA = states states = [] single3 = chain(states, any(states, notGroupStr(states, "'\\")), @@ -230,7 +234,9 @@ notChainStr(states, "''"))), any(states, notGroupStr(states, "'\\")))), chainStr(states, "'''")) - single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3)) + states, accepts = nfaToDfa(states, *single3) + single3DFA = NonGreedyDFA(states, accepts) + states_single3DFA = states states = [] double3 = chain(states, any(states, notGroupStr(states, '"\\')), @@ -245,9 +251,11 @@ notChainStr(states, '""'))), any(states, notGroupStr(states, '"\\')))), chainStr(states, '"""')) - double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3)) - map = {"'" : singleDFA, - '"' : doubleDFA, + states, accepts = nfaToDfa(states, *double3) + double3DFA = NonGreedyDFA(states, accepts) + states_double3DFA = states + map = {"'" : (singleDFA, states_singleDFA), + '"' : (doubleDFA, states_doubleDFA), "r" : None, "R" : None, "u" : None, @@ -257,25 +265,30 @@ for uniPrefix in ("", "u", "U", "b", "B", ): for rawPrefix in ("", "r", "R"): prefix = uniPrefix + rawPrefix - map[prefix + "'''"] = single3DFA - map[prefix + '"""'] = double3DFA + map[prefix + "'''"] = (single3DFA, states_single3DFA) + map[prefix + '"""'] = (double3DFA, states_double3DFA) return map # ______________________________________________________________________ -def output(name, dfa_class, dfa): +def output(name, dfa_class, dfa, states): import textwrap + lines = [] i = 0 for line in textwrap.wrap(repr(dfa.accepts), width = 50): if i == 0: - print "accepts =", line + lines.append("accepts = ") else: - print " ", line + lines.append(" ") + lines.append(line) + lines.append("\n") i += 1 import StringIO - print "states = [" - for numstate, state in enumerate(dfa.states): - print " #", numstate + lines.append("states = [\n") + for numstate, state in enumerate(states): + lines.append(" # ") + lines.append(str(numstate)) + lines.append('\n') s = StringIO.StringIO() i = 0 for k, v in sorted(state.items()): @@ -298,22 +311,28 @@ for line in text: line = line.replace('::', ': ') if i == 0: - print ' {' + line + lines.append(' {') else: - print ' ' + line + lines.append(' ') + lines.append(line) + lines.append('\n') i += 1 - print " ]" - print "%s = automata.%s(states, accepts)" % (name, dfa_class) - print + lines.append(" ]\n") + lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class)) + return ''.join(lines) def main (): - pseudoDFA = makePyPseudoDFA() - output("pseudoDFA", "DFA", pseudoDFA) + pseudoDFA, states_pseudoDFA = makePyPseudoDFA() + print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA) endDFAMap = makePyEndDFAMap() - output("double3DFA", "NonGreedyDFA", endDFAMap['"""']) - output("single3DFA", "NonGreedyDFA", endDFAMap["'''"]) - output("singleDFA", "DFA", endDFAMap["'"]) - output("doubleDFA", "DFA", endDFAMap['"']) + dfa, states = endDFAMap['"""'] + print output("double3DFA", "NonGreedyDFA", dfa, states) + dfa, states = endDFAMap["'''"] + print output("single3DFA", "NonGreedyDFA", dfa, states) + dfa, states = endDFAMap["'"] + print output("singleDFA", "DFA", dfa, states) + dfa, states = endDFAMap["\""] + print output("doubleDFA", "DFA", dfa, states) # ______________________________________________________________________ diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_gendfa.py @@ -0,0 +1,16 @@ +from pypy.interpreter.pyparser.automata import DFA, DEFAULT +from pypy.interpreter.pyparser.genpytokenize import output + +def test_states(): + states = [{"\x00": 1}, {"\x01": 0}] + d = DFA(states[:], [False, True]) + assert output('test', DFA, d, states) == """\ +accepts = [False, True] +states = [ + # 0 + {'\\x00': 1}, + # 1 + {'\\x01': 0}, + ] +test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts) +""" From pypy.commits at gmail.com Tue Jun 7 09:15:40 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:15:40 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: (s390x) impl uint_mul_high, removed uint_floordiv, int_floordiv and int_mod Message-ID: <5756c8fc.d81b1c0a.2d550.ffffcf68@mx.google.com> Author: Richard Plangger Branch: release-5.x Changeset: r84999:4dbea4b610dd Date: 2016-06-06 23:06 +0200 http://bitbucket.org/pypy/pypy/changeset/4dbea4b610dd/ Log: (s390x) impl uint_mul_high, removed uint_floordiv, int_floordiv and int_mod diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -439,7 +439,7 @@ prepare_int_lshift = helper.prepare_binary_op prepare_int_rshift = helper.prepare_binary_op prepare_uint_rshift = helper.prepare_binary_op - prepare_uint_mul_high = helper.prepare_binary_op + prepare_uint_mul_high = helper.prepare_int_mul_ovf prepare_int_add_ovf = helper.prepare_binary_op prepare_int_sub_ovf = helper.prepare_binary_op diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py --- a/rpython/jit/backend/zarch/instructions.py +++ b/rpython/jit/backend/zarch/instructions.py @@ -29,6 +29,7 @@ 'MGHI': ('ri', ['\xA7','\x0D']), 'MSGFI': ('ril', ['\xC2','\x00']), 'MLGR': ('rre', ['\xB9','\x86'], 'eo,r'), + 'MLG': ('rxy', ['\xE3','\x86'], 'eo,bid'), # div/mod 'DSGR': ('rre', ['\xB9','\x0D'], 'eo,r'), 'DSG': ('rxy', ['\xE3','\x0D'], 'eo,bidl'), diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -160,11 +160,15 @@ omc.BRC(c.ANY, l.imm(label_end - jmp_neither_lqlr_overflow)) omc.overwrite() - emit_int_floordiv = gen_emit_div_mod('DSGR', 'DSG') - emit_uint_floordiv = gen_emit_div_mod('DLGR', 'DLG') - # NOTE division sets one register with the modulo value, thus - # the regalloc ensures the right register survives. - emit_int_mod = gen_emit_div_mod('DSGR', 'DSG') + def emit_uint_mul_high(self, op, arglocs, regalloc): + r0, _, a1 = arglocs + # _ carries the value, contents of r0 are ignored + assert not r0.is_imm() + assert not a1.is_imm() + if a1.is_core_reg(): + self.mc.MLGR(r0, a1) + else: + self.mc.MLG(r0, a1) def emit_int_invert(self, op, arglocs, regalloc): l0, = arglocs diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -733,9 +733,6 @@ prepare_int_sub_ovf = helper.prepare_int_sub prepare_int_mul = helper.prepare_int_mul prepare_int_mul_ovf = helper.prepare_int_mul_ovf - prepare_int_floordiv = helper.prepare_int_div - prepare_uint_floordiv = helper.prepare_int_div - prepare_int_mod = helper.prepare_int_mod prepare_nursery_ptr_increment = prepare_int_add prepare_int_and = helper.prepare_int_logic @@ -746,6 +743,18 @@ prepare_int_lshift = helper.prepare_int_shift prepare_uint_rshift = helper.prepare_int_shift + def prepare_uint_mul_high(self, op): + a0 = op.getarg(0) + a1 = op.getarg(1) + if a0.is_constant(): + a0, a1 = a1, a0 + if helper.check_imm32(a1): + l1 = self.ensure_reg(a1) + else: + l1 = self.ensure_reg_or_pool(a1) + lr,lq = self.rm.ensure_even_odd_pair(a0, op, bind_first=True) + return [lr, lq, l1] + prepare_int_le = helper.generate_cmp_op() prepare_int_lt = helper.generate_cmp_op() prepare_int_ge = helper.generate_cmp_op() From pypy.commits at gmail.com Tue Jun 7 09:15:42 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:15:42 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: (s390x) killed some test that where specific to the three removed resoperations Message-ID: <5756c8fe.07a6c20a.9c49c.ffffd89c@mx.google.com> Author: Richard Plangger Branch: release-5.x Changeset: r85000:1d9b4894e55b Date: 2016-06-06 23:09 +0200 http://bitbucket.org/pypy/pypy/changeset/1d9b4894e55b/ Log: (s390x) killed some test that where specific to the three removed resoperations diff --git a/rpython/jit/backend/zarch/test/test_int.py b/rpython/jit/backend/zarch/test/test_int.py --- a/rpython/jit/backend/zarch/test/test_int.py +++ b/rpython/jit/backend/zarch/test/test_int.py @@ -35,41 +35,13 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail == finishdescr # ensures that guard is not taken! - def test_double_evenodd_pair(self): - code = """ - [i0] - i1 = int_floordiv(i0, 2) - i2 = int_floordiv(i0, 3) - i3 = int_floordiv(i0, 4) - i4 = int_floordiv(i0, 5) - i5 = int_floordiv(i0, 6) - i6 = int_floordiv(i0, 7) - i7 = int_floordiv(i0, 8) - i8 = int_le(i1, 0) - guard_true(i8) [i1,i2,i3,i4,i5,i6,i7] - finish(i0, descr=faildescr) - """ - # the guard forces 3 spills because after 4 divisions - # all even slots of the managed registers are full - loop = parse(code, namespace={'faildescr': BasicFinalDescr(1)}) - looptoken = JitCellToken() - self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - deadframe = self.cpu.execute_token(looptoken, 100) - fail = self.cpu.get_latest_descr(deadframe) - for i in range(2,9): - assert self.cpu.get_int_value(deadframe, i-2) == 100//i - - - @py.test.mark.parametrize('value', [2,3,15,2**16]) def test_evenodd_pair_extensive(self, value): instrs = [] failargs = [] values = [] j = 0 - mapping = (('int_floordiv',lambda x,y: x // y), - ('int_mod', lambda x,y: x % y), - ('int_mul_ovf', lambda x,y: x * y)) + mapping = (('int_mul_ovf', lambda x,y: x * y),) for i in range(20): name, func = mapping[j] instrs.append("i{d} = {i}(i0, {d})".format(d=i+1, i=name)) From pypy.commits at gmail.com Tue Jun 7 09:15:43 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Jun 2016 06:15:43 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: (s390x) removed two custom s390x tests that was using floordiv and mod Message-ID: <5756c8ff.2450c20a.b8f3a.0ce2@mx.google.com> Author: Richard Plangger Branch: release-5.x Changeset: r85001:c09c19272c99 Date: 2016-06-07 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/c09c19272c99/ Log: (s390x) removed two custom s390x tests that was using floordiv and mod diff --git a/rpython/jit/backend/zarch/test/test_regalloc.py b/rpython/jit/backend/zarch/test/test_regalloc.py --- a/rpython/jit/backend/zarch/test/test_regalloc.py +++ b/rpython/jit/backend/zarch/test/test_regalloc.py @@ -146,128 +146,3 @@ assert cpu.get_int_value(deadframe, 0) == 0 assert cpu.get_int_value(deadframe, 1) == -1000 -def test_bug_0(): - cpu, deadframe = run([-13, 10, 10, 8, -8, -16, -18, 46, -12, 26], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_gt(i3, -48) - i12 = int_xor(i8, i1) - i13 = int_gt(i6, -9) - i14 = int_le(i13, i2) - i15 = int_le(i11, i5) - i16 = uint_ge(i13, i13) - i17 = int_or(i9, -23) - i18 = int_lt(i10, i13) - i19 = int_or(i15, i5) - i20 = int_xor(i17, 54) - i21 = int_mul(i8, i10) - i22 = int_or(i3, i9) - i41 = int_and(i11, -4) - i42 = int_or(i41, 1) - i23 = int_mod(i12, i42) - i24 = int_is_true(i6) - i25 = uint_rshift(i15, 6) - i26 = int_or(-4, i25) - i27 = int_invert(i8) - i28 = int_sub(-113, i11) - i29 = int_neg(i7) - i30 = int_neg(i24) - i31 = int_floordiv(i3, 53) - i32 = int_mul(i28, i27) - i43 = int_and(i18, -4) - i44 = int_or(i43, 1) - i33 = int_mod(i26, i44) - i34 = int_or(i27, i19) - i35 = uint_lt(i13, 1) - i45 = int_and(i21, 31) - i36 = int_rshift(i21, i45) - i46 = int_and(i20, 31) - i37 = uint_rshift(i4, i46) - i38 = uint_gt(i33, -11) - i39 = int_neg(i7) - i40 = int_gt(i24, i32) - i99 = same_as_i(0) - guard_true(i99) [i40, i36, i37, i31, i16, i34, i35, i23, i22, i29, i14, i39, i30, i38] - finish(42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 0 - assert cpu.get_int_value(deadframe, 2) == 0 - assert cpu.get_int_value(deadframe, 3) == 0 - assert cpu.get_int_value(deadframe, 4) == 1 - assert cpu.get_int_value(deadframe, 5) == -7 - assert cpu.get_int_value(deadframe, 6) == 1 - assert cpu.get_int_value(deadframe, 7) == 0 - assert cpu.get_int_value(deadframe, 8) == -2 - assert cpu.get_int_value(deadframe, 9) == 18 - assert cpu.get_int_value(deadframe, 10) == 1 - assert cpu.get_int_value(deadframe, 11) == 18 - assert cpu.get_int_value(deadframe, 12) == -1 - assert cpu.get_int_value(deadframe, 13) == 0 - -def test_bug_1(): - cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], ''' - [i1, i2, i3, i4, i5, i6, i7, i8, i9, i10] - i11 = uint_lt(i6, 0) - i41 = int_and(i3, 31) - i12 = int_rshift(i3, i41) - i13 = int_neg(i2) - i14 = int_add(i11, i7) - i15 = int_or(i3, i2) - i16 = int_or(i12, i12) - i17 = int_ne(i2, i5) - i42 = int_and(i5, 31) - i18 = uint_rshift(i14, i42) - i43 = int_and(i14, 31) - i19 = int_lshift(7, i43) - i20 = int_neg(i19) - i21 = int_mod(i3, 1) - i22 = uint_ge(i15, i1) - i44 = int_and(i16, 31) - i23 = int_lshift(i8, i44) - i24 = int_is_true(i17) - i45 = int_and(i5, 31) - i25 = int_lshift(i14, i45) - i26 = int_lshift(i5, 17) - i27 = int_eq(i9, i15) - i28 = int_ge(0, i6) - i29 = int_neg(i15) - i30 = int_neg(i22) - i31 = int_add(i7, i16) - i32 = uint_lt(i19, i19) - i33 = int_add(i2, 1) - i34 = int_neg(i5) - i35 = int_add(i17, i24) - i36 = uint_lt(2, i16) - i37 = int_neg(i9) - i38 = int_gt(i4, i11) - i39 = int_lt(i27, i22) - i40 = int_neg(i27) - i99 = same_as_i(0) - guard_true(i99) [i40, i10, i36, i26, i13, i30, i21, i33, i18, i25, i31, i32, i28, i29, i35, i38, i20, i39, i34, i23, i37] - finish(-42) - ''') - assert cpu.get_int_value(deadframe, 0) == 0 - assert cpu.get_int_value(deadframe, 1) == 8 - assert cpu.get_int_value(deadframe, 2) == 1 - assert cpu.get_int_value(deadframe, 3) == 131072 - assert cpu.get_int_value(deadframe, 4) == 20 - assert cpu.get_int_value(deadframe, 5) == -1 - assert cpu.get_int_value(deadframe, 6) == 0 - assert cpu.get_int_value(deadframe, 7) == -19 - assert cpu.get_int_value(deadframe, 8) == 6 - assert cpu.get_int_value(deadframe, 9) == 26 - assert cpu.get_int_value(deadframe, 10) == 12 - assert cpu.get_int_value(deadframe, 11) == 0 - assert cpu.get_int_value(deadframe, 12) == 0 - assert cpu.get_int_value(deadframe, 13) == 2 - assert cpu.get_int_value(deadframe, 14) == 2 - assert cpu.get_int_value(deadframe, 15) == 1 - assert cpu.get_int_value(deadframe, 16) == -57344 - assert cpu.get_int_value(deadframe, 17) == 1 - assert cpu.get_int_value(deadframe, 18) == -1 - if WORD == 4: - assert cpu.get_int_value(deadframe, 19) == -2147483648 - elif WORD == 8: - assert cpu.get_int_value(deadframe, 19) == 19327352832 - assert cpu.get_int_value(deadframe, 20) == -49 - From pypy.commits at gmail.com Tue Jun 7 12:45:06 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 07 Jun 2016 09:45:06 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: Fix test_compiler.py Message-ID: <5756fa12.4fa81c0a.848ae.7991@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r85002:2401420c43db Date: 2016-06-07 17:44 +0100 http://bitbucket.org/pypy/pypy/changeset/2401420c43db/ Log: Fix test_compiler.py Make sure testing happens during the testing phase: the deprecated "generator tests" are executed during collection and should not do any testing themselves but merely yield a test function. diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -458,14 +458,17 @@ decl = str(decl) + "\n" yield self.st, decl, 'x', (1, 2, 3, 4) + def test_closure_error(self): source = """if 1: def f(a): del a def x(): a """ - exc = py.test.raises(SyntaxError, self.run, source).value - assert exc.msg == "Can't delete variable used in nested scopes: 'a'" + with py.test.raises(SyntaxError) as excinfo: + self.run(source) + msg = excinfo.value.msg + assert msg == "Can't delete variable used in nested scopes: 'a'" def test_try_except_finally(self): yield self.simple_test, """ @@ -879,7 +882,20 @@ """ self.simple_test(source, 'ok', 1) - def test_remove_docstring(self): + @py.test.mark.parametrize('expr, result', [ + ("f1.__doc__", None), + ("f2.__doc__", 'docstring'), + ("f2()", 'docstring'), + ("f3.__doc__", None), + ("f3()", 'bar'), + ("C1.__doc__", None), + ("C2.__doc__", 'docstring'), + ("C3.field", 'not docstring'), + ("C4.field", 'docstring'), + ("C4.__doc__", 'docstring'), + ("C4.__doc__", 'docstring'), + ("__doc__", None),]) + def test_remove_docstring(self, expr, result): source = '"module_docstring"\n' + """if 1: def f1(): 'docstring' @@ -903,19 +919,7 @@ code_w.remove_docstrings(self.space) dict_w = self.space.newdict(); code_w.exec_code(self.space, dict_w, dict_w) - - yield self.check, dict_w, "f1.__doc__", None - yield self.check, dict_w, "f2.__doc__", 'docstring' - yield self.check, dict_w, "f2()", 'docstring' - yield self.check, dict_w, "f3.__doc__", None - yield self.check, dict_w, "f3()", 'bar' - yield self.check, dict_w, "C1.__doc__", None - yield self.check, dict_w, "C2.__doc__", 'docstring' - yield self.check, dict_w, "C3.field", 'not docstring' - yield self.check, dict_w, "C4.field", 'docstring' - yield self.check, dict_w, "C4.__doc__", 'docstring' - yield self.check, dict_w, "C4.__doc__", 'docstring' - yield self.check, dict_w, "__doc__", None + self.check(dict_w, expr, result) def test_assert_skipping(self): space = self.space @@ -1111,7 +1115,7 @@ return d['f'](5) """) assert 'generator' in space.str_w(space.repr(w_generator)) - + def test_list_comprehension(self): source = "def f(): [i for i in l]" source2 = "def f(): [i for i in l for j in l]" From pypy.commits at gmail.com Tue Jun 7 13:25:50 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Jun 2016 10:25:50 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: By default, compile a single executable with both recording and Message-ID: <5757039e.e778c20a.56641.6920@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85003:5fd1d94677cf Date: 2016-06-07 19:26 +0200 http://bitbucket.org/pypy/pypy/changeset/5fd1d94677cf/ Log: By default, compile a single executable with both recording and replaying built in. Can compile a slightly faster single version by #defining rpy_rdb_replay to either 0 or 1. diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -450,6 +450,11 @@ T = self.lltypemap(op.args[-1]) if T is Void: result = '/* %s */' % result + if self.db.reverse_debugger: + S = self.lltypemap(op.args[0]).TO + if S._gckind != 'gc' and not S._hints.get('is_excdata'): + from rpython.translator.revdb import revdb_genc + result = revdb_genc.emit_void(result) return result def OP_GETFIELD(self, op, ampersand=''): diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -1,5 +1,6 @@ #include "common_header.h" #include +#include #include #include #include @@ -14,20 +15,59 @@ static char rpy_rev_buffer[16384]; static int rpy_rev_fileno = -1; +#ifndef rpy_rdb_replay +bool_t rpy_rdb_replay; +#endif -/* ------------------------------------------------------------ */ -#ifndef RPY_RDB_REPLAY -/* ------------------------------------------------------------ */ +static void setup_record_mode(int argc, char *argv[]); +static void setup_replay_mode(int *argc_p, char **argv_p[]); +static void check_at_end(void); RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) { /* init-time setup */ + int replay_asked = (*argc_p >= 2 && !strcmp((*argv_p)[1], "--replay")); + +#ifdef rpy_rdb_replay + if (replay_asked != rpy_rdb_replay) { + fprintf(stderr, "This executable was only compiled for %s mode.", + rpy_rdb_replay ? "replay" : "record"); + exit(1); + } +#else + rpy_rdb_replay = replay_asked; +#endif + + if (rpy_rdb_replay) + setup_replay_mode(argc_p, argv_p); + else + setup_record_mode(*argc_p, *argv_p); +} + +RPY_EXTERN +void rpy_reverse_db_teardown(void) +{ + if (!rpy_rdb_replay) + rpy_reverse_db_flush(); + else + check_at_end(); +} + + +/* ------------------------------------------------------------ */ +/* Recording mode */ +/* ------------------------------------------------------------ */ + + +static void setup_record_mode(int argc, char *argv[]) +{ char *filename = getenv("PYPYREVDB"); Signed x; + assert(!rpy_rdb_replay); rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; @@ -43,18 +83,12 @@ atexit(rpy_reverse_db_flush); } - RPY_REVDB_EMIT(x = RDB_SIGNATURE;, Signed _e, x); - RPY_REVDB_EMIT(x = RDB_VERSION;, Signed _e, x); - RPY_REVDB_EMIT(x = 0;, Signed _e, x); - RPY_REVDB_EMIT(x = 0;, Signed _e, x); - RPY_REVDB_EMIT(x = *argc_p;, Signed _e, x); - RPY_REVDB_EMIT(x = (Signed)*argv_p;, Signed _e, x); -} - -RPY_EXTERN -void rpy_reverse_db_teardown(void) -{ - rpy_reverse_db_flush(); + RPY_REVDB_EMIT(x = RDB_SIGNATURE; , Signed _e, x); + RPY_REVDB_EMIT(x = RDB_VERSION; , Signed _e, x); + RPY_REVDB_EMIT(x = 0; , Signed _e, x); + RPY_REVDB_EMIT(x = 0; , Signed _e, x); + RPY_REVDB_EMIT(x = argc; , Signed _e, x); + RPY_REVDB_EMIT(x = (Signed)argv; , Signed _e, x); } RPY_EXTERN @@ -88,56 +122,55 @@ /* ------------------------------------------------------------ */ -#else +/* Replaying mode */ /* ------------------------------------------------------------ */ -RPY_EXTERN -void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) +static void setup_replay_mode(int *argc_p, char **argv_p[]) { Signed x; + char *filename = (*argc_p) <= 2 ? "-" : (*argv_p)[2]; - if (*argc_p <= 1) { + if (!strcmp(filename, "-")) { rpy_rev_fileno = 0; /* stdin */ } else { - char *filename = (*argv_p)[1]; rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY); if (rpy_rev_fileno < 0) { fprintf(stderr, "Can't open file '%s': %m\n", filename); exit(1); } } + assert(rpy_rdb_replay); rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; - RPY_REVDB_EMIT(*, Signed _e, x); + RPY_REVDB_EMIT(abort();, Signed _e, x); if (x != RDB_SIGNATURE) { fprintf(stderr, "stdin is not a RevDB file (or wrong platform)\n"); exit(1); } - RPY_REVDB_EMIT(*, Signed _e, x); + RPY_REVDB_EMIT(abort();, Signed _e, x); if (x != RDB_VERSION) { fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n", (long)x, (long)RDB_VERSION); exit(1); } - RPY_REVDB_EMIT(*, Signed _e, x); /* ignored */ - RPY_REVDB_EMIT(*, Signed _e, x); /* ignored */ + RPY_REVDB_EMIT(abort();, Signed _e, x); /* ignored */ + RPY_REVDB_EMIT(abort();, Signed _e, x); /* ignored */ - RPY_REVDB_EMIT(*, Signed _e, x); + RPY_REVDB_EMIT(abort();, Signed _e, x); if (x <= 0) { fprintf(stderr, "RevDB file is bogus\n"); exit(1); } *argc_p = x; - RPY_REVDB_EMIT(*, Signed _e, x); + RPY_REVDB_EMIT(abort();, Signed _e, x); *argv_p = (char **)x; } -RPY_EXTERN -void rpy_reverse_db_teardown(void) +static void check_at_end(void) { char dummy[1]; if (rpy_revdb.buf_p != rpy_revdb.buf_limit || @@ -145,6 +178,7 @@ fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); exit(1); } + printf("Replaying finished.\n"); } RPY_EXTERN @@ -177,5 +211,3 @@ /* ------------------------------------------------------------ */ -#endif -/* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/revdb/rdb-src/revdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -7,51 +7,48 @@ RPY_EXTERN rpy_revdb_t rpy_revdb; -/* ------------------------------------------------------------ */ -#ifndef RPY_RDB_REPLAY +/* By default, this makes an executable which supports both recording + and replaying. It should help avoid troubles like using for + replaying an executable that is slightly different than the one + used for recording. In theory you can compile with + -Drpy_rdb_replay=0 or -Drpy_rdb_replay=1 to get only one version + compiled it (not tested so far). +*/ +#ifndef rpy_rdb_replay +RPY_EXTERN bool_t rpy_rdb_replay; +#endif + + /* ------------------------------------------------------------ */ -/* recording version of the macros */ #define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ + if (!rpy_rdb_replay) { \ normal_code \ - do { \ + { \ decl_e = variable; \ memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ rpy_reverse_db_flush(); \ - } while (0) + } \ + } else { \ + decl_e; \ + char *_src = rpy_revdb.buf_p; \ + char *_end1 = _src + sizeof(_e); \ + if (_end1 > rpy_revdb.buf_limit) { \ + _src = rpy_reverse_db_fetch(sizeof(_e)); \ + _end1 = _src + sizeof(_e); \ + } \ + rpy_revdb.buf_p = _end1; \ + memcpy(&_e, _src, sizeof(_e)); \ + variable = _e; \ + } + #define RPY_REVDB_EMIT_VOID(normal_code) \ - normal_code + if (!rpy_rdb_replay) { normal_code } else { } RPY_EXTERN void rpy_reverse_db_flush(void); +RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size); /* ------------------------------------------------------------ */ -#else -/* ------------------------------------------------------------ */ - - -/* replaying version of the macros */ -#define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ - do { \ - decl_e; \ - char *_src = rpy_revdb.buf_p; \ - char *_end1 = _src + sizeof(_e); \ - if (_end1 > rpy_revdb.buf_limit) { \ - _src = rpy_reverse_db_fetch(sizeof(_e)); \ - _end1 = _src + sizeof(_e); \ - } \ - rpy_revdb.buf_p = _end1; \ - memcpy(&_e, _src, sizeof(_e)); \ - variable = _e; \ - } while (0) -#define RPY_REVDB_EMIT_VOID(normal_code) \ - /* nothing */ - -RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size); - - -/* ------------------------------------------------------------ */ -#endif -/* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py --- a/rpython/translator/revdb/revdb_genc.py +++ b/rpython/translator/revdb/revdb_genc.py @@ -9,7 +9,10 @@ srcdir / 'revdb.c', ] +def emit_void(normal_code): + return 'RPY_REVDB_EMIT_VOID(%s);' % (normal_code,) + def emit(normal_code, tp, value): if tp == 'void @': - return 'RPY_REVDB_EMIT_VOID(%s);' % (normal_code,) + return emit_void(normal_code) return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value) diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -87,5 +87,6 @@ # assert got == [self.exename, 'abc', 'd'] # - # Now try the replay mode - xxx + # Now try the replay mode (just "doesn't crash" for now) + stdout = fn("--replay '%s'" % (self.rdbname,)) + assert stdout == "Replaying finished.\n" From pypy.commits at gmail.com Tue Jun 7 13:56:08 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 07 Jun 2016 10:56:08 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: fix test_sock_app.py Message-ID: <57570ab8.e4b3c20a.35934.ffff85c3@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r85004:449e6ff30c99 Date: 2016-06-07 18:32 +0100 http://bitbucket.org/pypy/pypy/changeset/449e6ff30c99/ Log: fix test_sock_app.py diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -671,13 +671,11 @@ class AppTestSocketTCP: HOST = 'localhost' - - def setup_class(cls): - cls.space = space + spaceconfig = {'usemodules': ['_socket', 'array']} def setup_method(self, method): - w_HOST = space.wrap(self.HOST) - self.w_serv = space.appexec([w_socket, w_HOST], + w_HOST = self.space.wrap(self.HOST) + self.w_serv =self.space.appexec([w_socket, w_HOST], '''(_socket, HOST): serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) serv.bind((HOST, 0)) @@ -687,7 +685,7 @@ def teardown_method(self, method): if hasattr(self, 'w_serv'): - space.appexec([self.w_serv], '(serv): serv.close()') + self.space.appexec([self.w_serv], '(serv): serv.close()') self.w_serv = None def test_timeout(self): @@ -803,8 +801,7 @@ class AppTestErrno: - def setup_class(cls): - cls.space = space + spaceconfig = {'usemodules': ['_socket']} def test_errno(self): from socket import socket, AF_INET, SOCK_STREAM, error From pypy.commits at gmail.com Tue Jun 7 13:56:10 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 07 Jun 2016 10:56:10 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: fix test_posix2.py Message-ID: <57570aba.4118c20a.1235c.2bab@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r85005:0f4725c1ff59 Date: 2016-06-07 18:49 +0100 http://bitbucket.org/pypy/pypy/changeset/0f4725c1ff59/ Log: fix test_posix2.py diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -13,14 +13,15 @@ import sys import signal +USEMODULES = ['binascii', 'posix', 'struct', 'time'] +if os.name != 'nt': + USEMODULES += ['fcntl'] +else: + # On windows, os.popen uses the subprocess module + USEMODULES += ['_rawffi', 'thread', 'signal'] + def setup_module(mod): - usemodules = ['binascii', 'posix', 'struct', 'time'] - if os.name != 'nt': - usemodules += ['fcntl'] - else: - # On windows, os.popen uses the subprocess module - usemodules += ['_rawffi', 'thread', 'signal'] - mod.space = gettestobjspace(usemodules=usemodules) + mod.space = gettestobjspace(usemodules=USEMODULES) mod.path = udir.join('posixtestfile.txt') mod.path.write("this is a test") mod.path2 = udir.join('test_posix2-') @@ -49,9 +50,10 @@ class AppTestPosix: + spaceconfig = {'usemodules': USEMODULES} def setup_class(cls): - cls.space = space + space = cls.space cls.w_runappdirect = space.wrap(cls.runappdirect) cls.w_posix = space.appexec([], GET_POSIX) cls.w_path = space.wrap(str(path)) @@ -1145,14 +1147,10 @@ class AppTestEnvironment(object): def setup_class(cls): - cls.space = space - cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) - cls.w_os = space.appexec([], "(): import os; return os") cls.w_path = space.wrap(str(path)) def test_environ(self): - posix = self.posix - os = self.os + import posix assert posix.environ['PATH'] del posix.environ['PATH'] def fn(): posix.environ['PATH'] @@ -1160,7 +1158,7 @@ if hasattr(__import__(os.name), "unsetenv"): def test_unsetenv_nonexisting(self): - os = self.os + import os os.unsetenv("XYZABC") #does not raise try: os.environ["ABCABC"] @@ -1178,8 +1176,6 @@ class AppTestPosixUnicode: def setup_class(cls): - cls.space = space - cls.w_posix = space.appexec([], GET_POSIX) if cls.runappdirect: # Can't change encoding try: @@ -1198,22 +1194,25 @@ def test_stat_unicode(self): # test that passing unicode would not raise UnicodeDecodeError + import posix try: - self.posix.stat(u"ą") + posix.stat(u"ą") except OSError: pass def test_open_unicode(self): # Ensure passing unicode doesn't raise UnicodeEncodeError + import posix try: - self.posix.open(u"ą", self.posix.O_WRONLY) + posix.open(u"ą", posix.O_WRONLY) except OSError: pass def test_remove_unicode(self): # See 2 above ;) + import posix try: - self.posix.remove(u"ą") + posix.remove(u"ą") except OSError: pass From pypy.commits at gmail.com Tue Jun 7 14:07:28 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Jun 2016 11:07:28 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: emit stop_point Message-ID: <57570d60.138f1c0a.dd3ab.ffffa178@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85006:9bbdb3034f2d Date: 2016-06-07 20:08 +0200 http://bitbucket.org/pypy/pypy/changeset/9bbdb3034f2d/ Log: emit stop_point diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -564,6 +564,8 @@ # __________ instrumentation _________ 'instrument_count': LLOp(), + + 'revdb_stop_point': LLOp(), } # ***** Run test_lloperation after changes. ***** diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -350,6 +350,9 @@ SetErrorMode(old_mode) if res.returncode != 0: if expect_crash: + if type(expect_crash) is int and expect_crash != res.returncode: + raise Exception("Returned %d, but expected %d" % ( + res.returncode, expect_crash)) return res.out, res.err print >> sys.stderr, res.err raise Exception("Returned %d" % (res.returncode,)) diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -104,7 +104,7 @@ exitcode = STANDALONE_ENTRY_POINT(argc, argv); #ifdef RPY_REVERSE_DEBUGGER - rpy_reverse_db_teardown(); + rpy_reverse_db_teardown(&exitcode); #endif pypy_debug_alloc_results(); diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -22,7 +22,7 @@ static void setup_record_mode(int argc, char *argv[]); static void setup_replay_mode(int *argc_p, char **argv_p[]); -static void check_at_end(void); +static void check_at_end(int exitcode, int *exitcode_p); RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) @@ -48,12 +48,15 @@ } RPY_EXTERN -void rpy_reverse_db_teardown(void) +void rpy_reverse_db_teardown(int *exitcode_p) { + int exitcode; + RPY_REVDB_EMIT(exitcode = *exitcode_p; , int _e, exitcode); + if (!rpy_rdb_replay) rpy_reverse_db_flush(); else - check_at_end(); + check_at_end(exitcode, exitcode_p); } @@ -170,15 +173,21 @@ *argv_p = (char **)x; } -static void check_at_end(void) +static void check_at_end(int exitcode, int *exitcode_p) { char dummy[1]; + if (*exitcode_p != exitcode) { + fprintf(stderr, "Bogus exit code\n"); + exit(1); + } if (rpy_revdb.buf_p != rpy_revdb.buf_limit || read(rpy_rev_fileno, dummy, 1) > 0) { fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); exit(1); } printf("Replaying finished.\n"); + rpy_reverse_db_stop_point(0); + *exitcode_p = 0; } RPY_EXTERN @@ -209,5 +218,11 @@ return rpy_rev_buffer; } +RPY_EXTERN +void rpy_reverse_db_stop_point(long stop_point) +{ + printf("stop_point %ld\n", stop_point); +} + /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/revdb/rdb-src/revdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -1,7 +1,7 @@ #include RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); -RPY_EXTERN void rpy_reverse_db_teardown(void); +RPY_EXTERN void rpy_reverse_db_teardown(int *exitcode_p); typedef struct { char *buf_p, *buf_limit; } rpy_revdb_t; RPY_EXTERN rpy_revdb_t rpy_revdb; @@ -47,8 +47,12 @@ #define RPY_REVDB_EMIT_VOID(normal_code) \ if (!rpy_rdb_replay) { normal_code } else { } +#define OP_REVDB_STOP_POINT(stop_point, r) \ + if (rpy_rdb_replay) rpy_reverse_db_stop_point(stop_point); + RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size); +RPY_EXTERN void rpy_reverse_db_stop_point(long stop_point); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -1,9 +1,10 @@ import py -import os +import os, sys import array, struct from rpython.tool.udir import udir from rpython.translator.interactive import Translation from rpython.rlib.rarithmetic import LONG_BIT +from rpython.rlib import revdb class RDB(object): @@ -47,9 +48,16 @@ def run(*argv): env = os.environ.copy() env['PYPYREVDB'] = self.rdbname - stdout = t.driver.cbuilder.cmdexec(' '.join(argv), env=env) + stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env, + expect_crash=9) + print >> sys.stderr, stderr return stdout - return run + + def replay(): + stdout = t.driver.cbuilder.cmdexec("--replay '%s'" % self.rdbname) + return stdout + + return run, replay def fetch_rdb(self): return RDB(self.rdbname) @@ -57,8 +65,8 @@ def test_simple(self): def main(argv): print argv[1:] - return 0 - fn = self.getcompiled(main, [], backendopt=False) + return 9 + fn, replay = self.getcompiled(main, [], backendopt=False) assert fn('abc d') == '[abc, d]\n' rdb = self.fetch_rdb() assert rdb.argc == 3 @@ -82,11 +90,27 @@ # write() call x = rdb.next(); assert x == len('[abc, d]\n') x = rdb.next('i'); assert x == 0 # errno + x = rdb.next('i'); assert x == 9 # exitcode # that's all that should get from this simple example assert rdb.done() # assert got == [self.exename, 'abc', 'd'] # # Now try the replay mode (just "doesn't crash" for now) - stdout = fn("--replay '%s'" % (self.rdbname,)) - assert stdout == "Replaying finished.\n" + out = replay() + assert out == ("Replaying finished.\n" + "stop_point 0\n") + + def test_simple_interpreter(self): + def main(argv): + for op in argv[1:]: + revdb.stop_point(42) + print op + return 9 + fn, replay = self.getcompiled(main, [], backendopt=False) + assert fn('abc d') == 'abc\nd\n' + out = replay() + assert out == ("stop_point 42\n" + "stop_point 42\n" + "Replaying finished.\n" + "stop_point 0\n") From pypy.commits at gmail.com Tue Jun 7 14:36:37 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 07 Jun 2016 11:36:37 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: fix test__vmprof.py Message-ID: <57571435.c68e1c0a.b5ee1.4b8b@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r85007:9814e558d693 Date: 2016-06-07 19:14 +0100 http://bitbucket.org/pypy/pypy/changeset/9814e558d693/ Log: fix test__vmprof.py diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -3,8 +3,9 @@ from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): + spaceconfig = {'usemodules': ['_vmprof', 'struct']} + def setup_class(cls): - cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1'))) cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2'))) @@ -17,7 +18,7 @@ import struct, sys, gc WORD = struct.calcsize('l') - + def count(s): i = 0 count = 0 @@ -44,7 +45,7 @@ else: raise AssertionError(ord(s[i])) return count - + import _vmprof gc.collect() # try to make the weakref list deterministic gc.collect() # by freeing all dead code objects From pypy.commits at gmail.com Tue Jun 7 15:19:18 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 07 Jun 2016 12:19:18 -0700 (PDT) Subject: [pypy-commit] pypy testing-cleanup: Use pytest_collection_modifyitems to apply the interplevel marker Message-ID: <57571e36.e153c20a.64238.ffff9a18@mx.google.com> Author: Ronan Lamy Branch: testing-cleanup Changeset: r85008:ae78acb71286 Date: 2016-06-07 20:18 +0100 http://bitbucket.org/pypy/pypy/changeset/ae78acb71286/ Log: Use pytest_collection_modifyitems to apply the interplevel marker diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -76,6 +76,17 @@ def pytest_pycollect_makemodule(path, parent): return PyPyModule(path, parent) +def is_applevel(item): + from pypy.tool.pytest.apptest import AppTestFunction + return isinstance(item, AppTestFunction) + +def pytest_collection_modifyitems(config, items): + if config.option.runappdirect: + return + for item in items: + if isinstance(item, py.test.Function) and not is_applevel(item): + item.add_marker('interplevel') + class PyPyModule(py.test.collect.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py --- a/pypy/tool/pytest/inttest.py +++ b/pypy/tool/pytest/inttest.py @@ -6,13 +6,8 @@ from pypy.conftest import PyPyClassCollector -marker = py.test.mark.interplevel - - class IntTestFunction(py.test.collect.Function): - def __init__(self, *args, **kwargs): - super(IntTestFunction, self).__init__(*args, **kwargs) - self._request.applymarker(marker) + pass class IntInstanceCollector(py.test.collect.Instance): From pypy.commits at gmail.com Tue Jun 7 15:19:33 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Jun 2016 12:19:33 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: push and pull to get it to shape for the next step (which involves fork) Message-ID: <57571e45.08371c0a.7877d.5f2b@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85009:9734700e55f1 Date: 2016-06-07 21:20 +0200 http://bitbucket.org/pypy/pypy/changeset/9734700e55f1/ Log: push and pull to get it to shape for the next step (which involves fork) diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -15,14 +15,10 @@ static char rpy_rev_buffer[16384]; static int rpy_rev_fileno = -1; -#ifndef rpy_rdb_replay -bool_t rpy_rdb_replay; -#endif - static void setup_record_mode(int argc, char *argv[]); static void setup_replay_mode(int *argc_p, char **argv_p[]); -static void check_at_end(int exitcode, int *exitcode_p); +static void check_at_end(int exitcode, int *exitcode_p, uint64_t stop_points); RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) @@ -31,17 +27,17 @@ int replay_asked = (*argc_p >= 2 && !strcmp((*argv_p)[1], "--replay")); -#ifdef rpy_rdb_replay - if (replay_asked != rpy_rdb_replay) { +#ifdef RPY_RDB_DYNAMIC_REPLAY + RPY_RDB_REPLAY = replay_asked; +#else + if (replay_asked != RPY_RDB_REPLAY) { fprintf(stderr, "This executable was only compiled for %s mode.", - rpy_rdb_replay ? "replay" : "record"); + RPY_RDB_REPLAY ? "replay" : "record"); exit(1); } -#else - rpy_rdb_replay = replay_asked; #endif - if (rpy_rdb_replay) + if (RPY_RDB_REPLAY) setup_replay_mode(argc_p, argv_p); else setup_record_mode(*argc_p, *argv_p); @@ -51,12 +47,15 @@ void rpy_reverse_db_teardown(int *exitcode_p) { int exitcode; + uint64_t stop_points; RPY_REVDB_EMIT(exitcode = *exitcode_p; , int _e, exitcode); + RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; , + uint64_t _e, stop_points); - if (!rpy_rdb_replay) + if (!RPY_RDB_REPLAY) rpy_reverse_db_flush(); else - check_at_end(exitcode, exitcode_p); + check_at_end(exitcode, exitcode_p, stop_points); } @@ -70,7 +69,7 @@ char *filename = getenv("PYPYREVDB"); Signed x; - assert(!rpy_rdb_replay); + assert(RPY_RDB_REPLAY == 0); rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; @@ -132,19 +131,23 @@ static void setup_replay_mode(int *argc_p, char **argv_p[]) { Signed x; - char *filename = (*argc_p) <= 2 ? "-" : (*argv_p)[2]; + int argc = *argc_p; + char **argv = *argv_p; + char *filename; - if (!strcmp(filename, "-")) { - rpy_rev_fileno = 0; /* stdin */ + if (argc != 3) { + fprintf(stderr, "syntax: %s --replay \n", argv[0]); + exit(2); } - else { - rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY); - if (rpy_rev_fileno < 0) { - fprintf(stderr, "Can't open file '%s': %m\n", filename); - exit(1); - } + filename = argv[2]; + + rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY); + if (rpy_rev_fileno < 0) { + fprintf(stderr, "Can't open file '%s': %m\n", filename); + exit(1); } - assert(rpy_rdb_replay); + + assert(RPY_RDB_REPLAY == 1); rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; @@ -171,13 +174,15 @@ RPY_REVDB_EMIT(abort();, Signed _e, x); *argv_p = (char **)x; + + rpy_revdb.stop_point_break = 1; } -static void check_at_end(int exitcode, int *exitcode_p) +static void check_at_end(int exitcode, int *exitcode_p, uint64_t stop_points) { char dummy[1]; - if (*exitcode_p != exitcode) { - fprintf(stderr, "Bogus exit code\n"); + if (stop_points != rpy_revdb.stop_point_seen) { + fprintf(stderr, "Bad number of stop points\n"); exit(1); } if (rpy_revdb.buf_p != rpy_revdb.buf_limit || @@ -185,8 +190,12 @@ fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); exit(1); } - printf("Replaying finished.\n"); - rpy_reverse_db_stop_point(0); + if (*exitcode_p != exitcode) { + fprintf(stderr, "Bogus exit code\n"); + exit(1); + } + printf("Replaying finished (exit code %d)\n", exitcode); + rpy_reverse_db_break(0); *exitcode_p = 0; } @@ -219,9 +228,10 @@ } RPY_EXTERN -void rpy_reverse_db_stop_point(long stop_point) +void rpy_reverse_db_break(long stop_point) { - printf("stop_point %ld\n", stop_point); + printf("break #%ld after %lld stop points\n", stop_point, + (long long)rpy_revdb.stop_point_seen); } diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/revdb/rdb-src/revdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -1,29 +1,36 @@ #include +#include + +/* By default, this makes an executable which supports both recording + and replaying. It should help avoid troubles like using for + replaying an executable that is slightly different than the one + used for recording. In theory you can compile with + -DRPY_RDB_REPLAY=0 or -DRPY_RDB_REPLAY=1 to get only one version + compiled for it, which should be slightly faster (not tested so + far). +*/ + +typedef struct { +#ifndef RPY_RDB_REPLAY + bool_t replay; +#define RPY_RDB_REPLAY rpy_revdb.replay +#define RPY_RDB_DYNAMIC_REPLAY +#endif + char *buf_p, *buf_limit; + uint64_t stop_point_seen, stop_point_break; +} rpy_revdb_t; + +RPY_EXTERN rpy_revdb_t rpy_revdb; + + +/* ------------------------------------------------------------ */ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(int *exitcode_p); -typedef struct { char *buf_p, *buf_limit; } rpy_revdb_t; -RPY_EXTERN rpy_revdb_t rpy_revdb; - - -/* By default, this makes an executable which supports both recording - and replaying. It should help avoid troubles like using for - replaying an executable that is slightly different than the one - used for recording. In theory you can compile with - -Drpy_rdb_replay=0 or -Drpy_rdb_replay=1 to get only one version - compiled it (not tested so far). -*/ -#ifndef rpy_rdb_replay -RPY_EXTERN bool_t rpy_rdb_replay; -#endif - - -/* ------------------------------------------------------------ */ - #define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ - if (!rpy_rdb_replay) { \ + if (!RPY_RDB_REPLAY) { \ normal_code \ { \ decl_e = variable; \ @@ -45,14 +52,15 @@ } #define RPY_REVDB_EMIT_VOID(normal_code) \ - if (!rpy_rdb_replay) { normal_code } else { } + if (!RPY_RDB_REPLAY) { normal_code } else { } #define OP_REVDB_STOP_POINT(stop_point, r) \ - if (rpy_rdb_replay) rpy_reverse_db_stop_point(stop_point); + if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break) \ + rpy_reverse_db_break(stop_point); RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size); -RPY_EXTERN void rpy_reverse_db_stop_point(long stop_point); +RPY_EXTERN void rpy_reverse_db_break(long stop_point); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -25,6 +25,9 @@ self.cur = p + struct.calcsize(mode) return struct.unpack_from(mode, self.buffer, p)[0] + def number_of_stop_points(self): + return struct.unpack_from("q", self.buffer, len(self.buffer) - 8)[0] + def done(self): return self.cur == len(self.buffer) @@ -91,6 +94,7 @@ x = rdb.next(); assert x == len('[abc, d]\n') x = rdb.next('i'); assert x == 0 # errno x = rdb.next('i'); assert x == 9 # exitcode + x = rdb.next('q'); assert x == 0 # number of stop points # that's all that should get from this simple example assert rdb.done() # @@ -98,8 +102,8 @@ # # Now try the replay mode (just "doesn't crash" for now) out = replay() - assert out == ("Replaying finished.\n" - "stop_point 0\n") + assert out == ("Replaying finished (exit code 9)\n" + "break #0 after 0 stop points\n") def test_simple_interpreter(self): def main(argv): @@ -108,9 +112,9 @@ print op return 9 fn, replay = self.getcompiled(main, [], backendopt=False) - assert fn('abc d') == 'abc\nd\n' + assert fn('abc d ef') == 'abc\nd\nef\n' + assert self.fetch_rdb().number_of_stop_points() == 3 out = replay() - assert out == ("stop_point 42\n" - "stop_point 42\n" - "Replaying finished.\n" - "stop_point 0\n") + assert out == ("break #42 after 1 stop points\n" + "Replaying finished (exit code 9)\n" + "break #0 after 3 stop points\n") From pypy.commits at gmail.com Tue Jun 7 15:35:12 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 07 Jun 2016 12:35:12 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Remove starargs and kwargs in handle_classdef Message-ID: <575721f0.665ec20a.79856.ffff9b72@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85010:facd0617efcb Date: 2016-06-07 21:34 +0200 http://bitbucket.org/pypy/pypy/changeset/facd0617efcb/ Log: Remove starargs and kwargs in handle_classdef diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -452,12 +452,12 @@ if classdef_node.num_children() == 4: # class NAME ':' suite body = self.handle_suite(classdef_node.get_child(3)) - return ast.ClassDef(name, None, None, None, None, body, decorators, + return ast.ClassDef(name, None, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) if classdef_node.get_child(3).type == tokens.RPAR: # class NAME '(' ')' ':' suite body = self.handle_suite(classdef_node.get_child(5)) - return ast.ClassDef(name, None, None, None, None, body, decorators, + return ast.ClassDef(name, None, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) # class NAME '(' arglist ')' ':' suite @@ -467,7 +467,7 @@ call = self.handle_call(classdef_node.get_child(3), call_name) body = self.handle_suite(classdef_node.get_child(6)) return ast.ClassDef( - name, call.args, call.keywords, call.starargs, call.kwargs, + name, call.args, call.keywords, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) def handle_class_bases(self, bases_node): From pypy.commits at gmail.com Tue Jun 7 15:52:06 2016 From: pypy.commits at gmail.com (raffael_t) Date: Tue, 07 Jun 2016 12:52:06 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Remove 2 parameters for Call (requires working ast) Message-ID: <575725e6.c12bc20a.9d337.ffffa901@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85011:1c03b34c7f8c Date: 2016-06-07 21:51 +0200 http://bitbucket.org/pypy/pypy/changeset/1c03b34c7f8c/ Log: Remove 2 parameters for Call (requires working ast) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -511,7 +511,7 @@ if decorator_node.num_children() == 3: dec = dec_name elif decorator_node.num_children() == 5: - dec = ast.Call(dec_name, None, None, None, None, + dec = ast.Call(dec_name, None, None, decorator_node.get_lineno(), decorator_node.get_column()) else: dec = self.handle_call(decorator_node.get_child(3), dec_name) @@ -984,7 +984,7 @@ first_child = trailer_node.get_child(0) if first_child.type == tokens.LPAR: if trailer_node.num_children() == 2: - return ast.Call(left_expr, None, None, None, None, + return ast.Call(left_expr, None, None, trailer_node.get_lineno(), trailer_node.get_column()) else: return self.handle_call(trailer_node.get_child(1), left_expr) @@ -1017,6 +1017,7 @@ return ast.Subscript(left_expr, ast.Index(tup), ast.Load, middle.get_lineno(), middle.get_column()) + #fix this method def handle_call(self, args_node, callable_expr): arg_count = 0 keyword_count = 0 From pypy.commits at gmail.com Tue Jun 7 16:52:42 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Jun 2016 13:52:42 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Documentation (not implemented yet). Other clean-ups. Message-ID: <5757341a.03c31c0a.1ef99.6f3e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85012:0e59d36285f8 Date: 2016-06-07 22:53 +0200 http://bitbucket.org/pypy/pypy/changeset/0e59d36285f8/ Log: Documentation (not implemented yet). Other clean-ups. diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -104,7 +104,7 @@ exitcode = STANDALONE_ENTRY_POINT(argc, argv); #ifdef RPY_REVERSE_DEBUGGER - rpy_reverse_db_teardown(&exitcode); + rpy_reverse_db_teardown(); #endif pypy_debug_alloc_results(); diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c --- a/rpython/translator/revdb/rdb-src/revdb.c +++ b/rpython/translator/revdb/rdb-src/revdb.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -11,6 +12,14 @@ #define RDB_VERSION 0x00FF0001 +typedef struct { + Signed signature, version; + Signed reserved1, reserved2; + int argc; + char **argv; +} rdb_header_t; + + rpy_revdb_t rpy_revdb; static char rpy_rev_buffer[16384]; static int rpy_rev_fileno = -1; @@ -18,7 +27,7 @@ static void setup_record_mode(int argc, char *argv[]); static void setup_replay_mode(int *argc_p, char **argv_p[]); -static void check_at_end(int exitcode, int *exitcode_p, uint64_t stop_points); +static void check_at_end(uint64_t stop_points); RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]) @@ -44,18 +53,16 @@ } RPY_EXTERN -void rpy_reverse_db_teardown(int *exitcode_p) +void rpy_reverse_db_teardown(void) { - int exitcode; uint64_t stop_points; - RPY_REVDB_EMIT(exitcode = *exitcode_p; , int _e, exitcode); RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; , uint64_t _e, stop_points); if (!RPY_RDB_REPLAY) rpy_reverse_db_flush(); else - check_at_end(exitcode, exitcode_p, stop_points); + check_at_end(stop_points); } @@ -64,10 +71,27 @@ /* ------------------------------------------------------------ */ +static void write_all(int fd, const char *buf, ssize_t count) +{ + while (count > 0) { + ssize_t wsize = write(fd, buf, count); + if (wsize <= 0) { + if (wsize == 0) + fprintf(stderr, "Writing to PYPYREVDB file: " + "unexpected non-blocking mode\n"); + else + fprintf(stderr, "Fatal error: writing to PYPYREVDB file: %m\n"); + abort(); + } + buf += wsize; + count -= wsize; + } +} + static void setup_record_mode(int argc, char *argv[]) { char *filename = getenv("PYPYREVDB"); - Signed x; + rdb_header_t h; assert(RPY_RDB_REPLAY == 0); rpy_revdb.buf_p = rpy_rev_buffer; @@ -85,12 +109,12 @@ atexit(rpy_reverse_db_flush); } - RPY_REVDB_EMIT(x = RDB_SIGNATURE; , Signed _e, x); - RPY_REVDB_EMIT(x = RDB_VERSION; , Signed _e, x); - RPY_REVDB_EMIT(x = 0; , Signed _e, x); - RPY_REVDB_EMIT(x = 0; , Signed _e, x); - RPY_REVDB_EMIT(x = argc; , Signed _e, x); - RPY_REVDB_EMIT(x = (Signed)argv; , Signed _e, x); + memset(&h, 0, sizeof(h)); + h.signature = RDB_SIGNATURE; + h.version = RDB_VERSION; + h.argc = argc; + h.argv = argv; + write_all(rpy_rev_fileno, (const char *)&h, sizeof(h)); } RPY_EXTERN @@ -104,22 +128,7 @@ if (size == 0 || rpy_rev_fileno < 0) return; - p = rpy_rev_buffer; - retry: - wsize = write(rpy_rev_fileno, p, size); - if (wsize >= size) - return; - if (wsize <= 0) { - if (wsize == 0) - fprintf(stderr, "Writing to PYPYREVDB file: " - "unexpected non-blocking mode\n"); - else - fprintf(stderr, "Fatal error: writing to PYPYREVDB file: %m\n"); - abort(); - } - p += wsize; - size -= wsize; - goto retry; + write_all(rpy_rev_fileno, rpy_rev_buffer, size); } @@ -128,12 +137,87 @@ /* ------------------------------------------------------------ */ +/* How it works: the main process reads the RevDB file and + reconstructs the GC objects, in effect replaying their content, for + the complete duration of the original program. During this + replaying, it forks a fixed number of frozen processes which sit + around, each keeping the version of the GC objects contents at some + known version. We have one pipe for every frozen process, which + the frozen process is blocked reading. + + [main process] + [frozen process 1] + [frozen process 2] + [debugging process] + [frozen process 3] + [frozen process 4] + ... + [frozen process n] + + When all frozen processes are made, the main process enters + interactive mode. In interactive mode, the main process reads from + stdin a version number to go to. It picks the correct frozen + process (the closest one that is before in time); let's say it is + process #p. It sends the version number to it by writing to pipe + #p. The corresponding frozen process wakes up, and forks again + into a debugging process. The main and the frozen process then + block. + + The debugging process first goes forward in time until it reaches + the right version number. Then it interacts with the user (or a + pdb-like program outside) on stdin/stdout. This goes on until an + "exit" command is received and the debugging process dies. At that + point its parent (the frozen process) continues and signals its own + parent (the main process) by writing to a separate signalling pipe. + The main process then wakes up again, and the loop closes: it reads + on stdin the next version number that we're interested in, and asks + the right frozen process to make a debugging process again. + + Note how we have, over time, several processes that read and + process stdin; it should work because they are strictly doing that + in sequence, never concurrently. To avoid the case where stdin is + buffered inside one process but a different process should read it, + we write markers to stdout when such switches occur. The outside + controlling program must wait until it sees these markers before + writing more data. +*/ + +#define FROZEN_PROCESSES 30 +#define GOLDEN_RATIO 0.618034 + +static uint64_t total_stop_points; + + +static ssize_t read_at_least(int fd, char *buf, + ssize_t count_min, ssize_t count_max) +{ + ssize_t result = 0; + assert(count_min <= count_max); + while (result < count_min) { + ssize_t rsize = read(fd, buf + result, count_max - result); + if (rsize <= 0) { + if (rsize == 0) + fprintf(stderr, "RevDB file appears truncated\n"); + else + fprintf(stderr, "RevDB file read error: %m\n"); + exit(1); + } + result += rsize; + } + return result; +} + +static void read_all(int fd, char *buf, ssize_t count) +{ + (void)read_at_least(fd, buf, count, count); +} + static void setup_replay_mode(int *argc_p, char **argv_p[]) { - Signed x; int argc = *argc_p; char **argv = *argv_p; char *filename; + rdb_header_t h; if (argc != 3) { fprintf(stderr, "syntax: %s --replay \n", argv[0]); @@ -148,37 +232,51 @@ } assert(RPY_RDB_REPLAY == 1); + + read_all(rpy_rev_fileno, (char *)&h, sizeof(h)); + + if (h.signature != RDB_SIGNATURE) { + fprintf(stderr, "'%s' is not a RevDB file (or wrong platform)\n", + filename); + exit(1); + } + if (h.version != RDB_VERSION) { + fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n", + (long)h.version, (long)RDB_VERSION); + exit(1); + } + *argc_p = h.argc; + *argv_p = h.argv; + + if (lseek(rpy_rev_fileno, -sizeof(uint64_t), SEEK_END) < 0 || + read(rpy_rev_fileno, &total_stop_points, + sizeof(uint64_t)) != sizeof(uint64_t) || + lseek(rpy_rev_fileno, sizeof(h), SEEK_SET) != sizeof(h)) { + fprintf(stderr, "%s: %m\n", filename); + exit(1); + } + rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; - - RPY_REVDB_EMIT(abort();, Signed _e, x); - if (x != RDB_SIGNATURE) { - fprintf(stderr, "stdin is not a RevDB file (or wrong platform)\n"); - exit(1); - } - RPY_REVDB_EMIT(abort();, Signed _e, x); - if (x != RDB_VERSION) { - fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n", - (long)x, (long)RDB_VERSION); - exit(1); - } - RPY_REVDB_EMIT(abort();, Signed _e, x); /* ignored */ - RPY_REVDB_EMIT(abort();, Signed _e, x); /* ignored */ - - RPY_REVDB_EMIT(abort();, Signed _e, x); - if (x <= 0) { - fprintf(stderr, "RevDB file is bogus\n"); - exit(1); - } - *argc_p = x; - - RPY_REVDB_EMIT(abort();, Signed _e, x); - *argv_p = (char **)x; - rpy_revdb.stop_point_break = 1; } -static void check_at_end(int exitcode, int *exitcode_p, uint64_t stop_points) +RPY_EXTERN +char *rpy_reverse_db_fetch(int expected_size) +{ + ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p; + assert(keep >= 0); + memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep); + rsize = read_at_least(rpy_rev_fileno, + rpy_rev_buffer + keep, + expected_size - keep, + sizeof(rpy_rev_buffer) - keep); + rpy_revdb.buf_p = rpy_rev_buffer; + rpy_revdb.buf_limit = rpy_rev_buffer + keep + rsize; + return rpy_rev_buffer; +} + +static void check_at_end(uint64_t stop_points) { char dummy[1]; if (stop_points != rpy_revdb.stop_point_seen) { @@ -190,41 +288,12 @@ fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); exit(1); } - if (*exitcode_p != exitcode) { - fprintf(stderr, "Bogus exit code\n"); + if (stop_points != total_stop_points) { + fprintf(stderr, "RevDB file modified while reading?\n"); exit(1); } - printf("Replaying finished (exit code %d)\n", exitcode); - rpy_reverse_db_break(0); - *exitcode_p = 0; -} - -RPY_EXTERN -char *rpy_reverse_db_fetch(int expected_size) -{ - ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p; - assert(keep >= 0); - memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep); - - retry: - rsize = read(rpy_rev_fileno, rpy_rev_buffer + keep, - sizeof(rpy_rev_buffer) - keep); - if (rsize <= 0) { - if (rsize == 0) - fprintf(stderr, "RevDB file appears truncated\n"); - else - fprintf(stderr, "RevDB file read error: %m\n"); - exit(1); - } - keep += rsize; - - rpy_revdb.buf_p = rpy_rev_buffer; - rpy_revdb.buf_limit = rpy_rev_buffer + keep; - - if (rpy_revdb.buf_limit - rpy_revdb.buf_p < expected_size) - goto retry; - - return rpy_rev_buffer; + printf("Replaying finished, %lld stop points\n", (long long)stop_points); + exit(0); } RPY_EXTERN diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h --- a/rpython/translator/revdb/rdb-src/revdb_include.h +++ b/rpython/translator/revdb/rdb-src/revdb_include.h @@ -26,7 +26,7 @@ /* ------------------------------------------------------------ */ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); -RPY_EXTERN void rpy_reverse_db_teardown(int *exitcode_p); +RPY_EXTERN void rpy_reverse_db_teardown(void); #define RPY_REVDB_EMIT(normal_code, decl_e, variable) \ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -93,17 +93,15 @@ # write() call x = rdb.next(); assert x == len('[abc, d]\n') x = rdb.next('i'); assert x == 0 # errno - x = rdb.next('i'); assert x == 9 # exitcode x = rdb.next('q'); assert x == 0 # number of stop points - # that's all that should get from this simple example + # that's all we should get from this simple example assert rdb.done() # assert got == [self.exename, 'abc', 'd'] # # Now try the replay mode (just "doesn't crash" for now) out = replay() - assert out == ("Replaying finished (exit code 9)\n" - "break #0 after 0 stop points\n") + assert out == "Replaying finished, 0 stop points\n" def test_simple_interpreter(self): def main(argv): @@ -116,5 +114,4 @@ assert self.fetch_rdb().number_of_stop_points() == 3 out = replay() assert out == ("break #42 after 1 stop points\n" - "Replaying finished (exit code 9)\n" - "break #0 after 3 stop points\n") + "Replaying finished, 3 stop points\n") From pypy.commits at gmail.com Tue Jun 7 17:02:03 2016 From: pypy.commits at gmail.com (mattip) Date: Tue, 07 Jun 2016 14:02:03 -0700 (PDT) Subject: [pypy-commit] pypy release-5.x: Added tag release-pypy2.7-v5.3 for changeset c09c19272c99 Message-ID: <5757364b.07a6c20a.9c49c.ffff8a22@mx.google.com> Author: Matti Picus Branch: release-5.x Changeset: r85014:e21678d5232d Date: 2016-06-07 23:59 +0300 http://bitbucket.org/pypy/pypy/changeset/e21678d5232d/ Log: Added tag release-pypy2.7-v5.3 for changeset c09c19272c99 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -25,3 +25,4 @@ 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 +c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 From pypy.commits at gmail.com Tue Jun 7 17:02:02 2016 From: pypy.commits at gmail.com (mattip) Date: Tue, 07 Jun 2016 14:02:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.3 for changeset c09c19272c99 Message-ID: <5757364a.c7b81c0a.71767.7734@mx.google.com> Author: Matti Picus Branch: Changeset: r85013:ff206a3e2f41 Date: 2016-06-07 23:59 +0300 http://bitbucket.org/pypy/pypy/changeset/ff206a3e2f41/ Log: Added tag release-pypy2.7-v5.3 for changeset c09c19272c99 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -25,3 +25,4 @@ 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 +c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 From pypy.commits at gmail.com Tue Jun 7 17:31:37 2016 From: pypy.commits at gmail.com (mattip) Date: Tue, 07 Jun 2016 14:31:37 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: add checksums Message-ID: <57573d39.86981c0a.7317.ffff8661@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r753:3d659e7fcecd Date: 2016-06-08 00:31 +0300 http://bitbucket.org/pypy/pypy.org/changeset/3d659e7fcecd/ Log: add checksums diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -412,6 +412,18 @@ 204273a21dbf71c0827966265c40eb7a pypy-5.1.0-src.zip a1710ae6f15b567bf3c8fd608553ad48 pypy-5.1.0-win32.zip +

pypy2.7-v5.3 md5:

+
+05078bcdd797a025223d5905e2a12332 pypy2-v5.3.0-linux32.tar.bz2 +7d01e12eaca473258801faebc2db12d8 pypy2-v5.3.0-linux64.tar.bz2 +bf1640973865c5ca1bc88e299455cbcc pypy2-v5.3.0-linux-armel.tar.bz2 +2aa5941a05d46427293d48d67d079df5 pypy2-v5.3.0-linux-armhf-raring.tar.bz2 +9c2cc832ba15fd4a08ba7e676226f406 pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2 +21a346cca4e8e6897381a0e647a86d68 pypy2-v5.3.0-osx64.tar.bz2 +c39a578078ab3145d2a584cacf4c164c pypy2-v5.3.0-s390x.tar.bz2 +45ce35a438ed8ae1539cc05407d43965 pypy2-v5.3.0-src.tar.bz2 +24add66f18ab2213c9e44af0ada61085 pypy2-v5.3.0-src.zip +f6197adf58bfa32bcb18451289da1c7c pypy2-v5.3.0-win32.zip

pypy3.3-v5.2-alpha md5:

 1176464541dff42e685bf8a9bb393796  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -483,6 +495,30 @@
 3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
 53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
+

pypy2.7-5.3 sha1

+
+401066f82c8a26dfb1e3421ae4b117655b55ee8d pypy2-v5.3.0-linux32.tar.bz2 +939a49319ed8e25ecb9f646ba50f0618eb52c99b pypy2-v5.3.0-linux64.tar.bz2 +00fe6e6d672c65d9096b45a708a1be95bca412c4 pypy2-v5.3.0-linux-armel.tar.bz2 +08fd47ffdf3bc9b7409147e9a0a576e0d577d735 pypy2-v5.3.0-linux-armhf-raring.tar.bz2 +1572762c5b76a6efda27110267f165bf9b78402d pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2 +6d6219b7d42b6f31118d2ace2280932dd52d4d9d pypy2-v5.3.0-osx64.tar.bz2 +1d4f5f547d798294755fc9e14e2b6490e8a5d194 pypy2-v5.3.0-s390x.tar.bz2 +7bdb1cfc604192fc2a39023588d648a30975f0e4 pypy2-v5.3.0-src.tar.bz2 +18a81c46b3d0ecf7e186c19a7d302986a5b15a83 pypy2-v5.3.0-src.zip +076251ba3b44435dc11867dab00f392b058bdc7c pypy2-v5.3.0-win32.zip
+

pypy2.7-5.3 sha256

+
+bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2 pypy2-v5.3.0-linux32.tar.bz2 +ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588 pypy2-v5.3.0-linux64.tar.bz2 +81b6f589a947d7353bb69408c46d4833d6e9cb501f3c3f0c73bd28d0e3df69aa pypy2-v5.3.0-linux-armel.tar.bz2 +bdb911a87e773a292334061b9c33b907f46d987e403fe94cc627a3b9b1c9cb19 pypy2-v5.3.0-linux-armhf-raring.tar.bz2 +87b3566b6bbb8bf31c2f0d72bf31d95142fdce004d987812336a59d788005bed pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2 +1b103bacbdcdbbc490660ec0c7b3d99d1ff1cfc2f13cd403db21c27f03d36a1d pypy2-v5.3.0-osx64.tar.bz2 +1ccc0ce08dd55d188d3971331020a1c82e917e418846d2c2c07a225733d85b1e pypy2-v5.3.0-s390x.tar.bz2 +4142eb8f403810bc88a4911792bb5a502e152df95806e33e69050c828cd160d5 pypy2-v5.3.0-src.tar.bz2 +09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58 pypy2-v5.3.0-src.zip +32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20 pypy2-v5.3.0-win32.zip

pypy3.3-v5.2-alpha sha1:

 03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -449,6 +449,19 @@
     204273a21dbf71c0827966265c40eb7a  pypy-5.1.0-src.zip
     a1710ae6f15b567bf3c8fd608553ad48  pypy-5.1.0-win32.zip
 
+pypy2.7-v5.3 md5:
+
+    05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
+    7d01e12eaca473258801faebc2db12d8  pypy2-v5.3.0-linux64.tar.bz2
+    bf1640973865c5ca1bc88e299455cbcc  pypy2-v5.3.0-linux-armel.tar.bz2
+    2aa5941a05d46427293d48d67d079df5  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
+    9c2cc832ba15fd4a08ba7e676226f406  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
+    21a346cca4e8e6897381a0e647a86d68  pypy2-v5.3.0-osx64.tar.bz2
+    c39a578078ab3145d2a584cacf4c164c  pypy2-v5.3.0-s390x.tar.bz2
+    45ce35a438ed8ae1539cc05407d43965  pypy2-v5.3.0-src.tar.bz2
+    24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
+    f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
+
 pypy3.3-v5.2-alpha md5::
 
     1176464541dff42e685bf8a9bb393796  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -521,6 +534,32 @@
     3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
     53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
 
+pypy2.7-5.3 sha1
+
+    401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
+    939a49319ed8e25ecb9f646ba50f0618eb52c99b  pypy2-v5.3.0-linux64.tar.bz2
+    00fe6e6d672c65d9096b45a708a1be95bca412c4  pypy2-v5.3.0-linux-armel.tar.bz2
+    08fd47ffdf3bc9b7409147e9a0a576e0d577d735  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
+    1572762c5b76a6efda27110267f165bf9b78402d  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
+    6d6219b7d42b6f31118d2ace2280932dd52d4d9d  pypy2-v5.3.0-osx64.tar.bz2
+    1d4f5f547d798294755fc9e14e2b6490e8a5d194  pypy2-v5.3.0-s390x.tar.bz2
+    7bdb1cfc604192fc2a39023588d648a30975f0e4  pypy2-v5.3.0-src.tar.bz2
+    18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
+    076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
+
+pypy2.7-5.3 sha256
+
+    bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
+    ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588  pypy2-v5.3.0-linux64.tar.bz2
+    81b6f589a947d7353bb69408c46d4833d6e9cb501f3c3f0c73bd28d0e3df69aa  pypy2-v5.3.0-linux-armel.tar.bz2
+    bdb911a87e773a292334061b9c33b907f46d987e403fe94cc627a3b9b1c9cb19  pypy2-v5.3.0-linux-armhf-raring.tar.bz2
+    87b3566b6bbb8bf31c2f0d72bf31d95142fdce004d987812336a59d788005bed  pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
+    1b103bacbdcdbbc490660ec0c7b3d99d1ff1cfc2f13cd403db21c27f03d36a1d  pypy2-v5.3.0-osx64.tar.bz2
+    1ccc0ce08dd55d188d3971331020a1c82e917e418846d2c2c07a225733d85b1e  pypy2-v5.3.0-s390x.tar.bz2
+    4142eb8f403810bc88a4911792bb5a502e152df95806e33e69050c828cd160d5  pypy2-v5.3.0-src.tar.bz2
+    09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
+    32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
+
 pypy3.3-v5.2-alpha sha1::
 
     03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2

From pypy.commits at gmail.com  Tue Jun  7 18:37:28 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 07 Jun 2016 15:37:28 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: kill
 pypy/tool/pytest/inttest.py and just use regular pytest objects for
 interp-level tests
Message-ID: <57574ca8.0c9c1c0a.100d6.ffff9260@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85015:eb94727607d8
Date: 2016-06-07 23:36 +0100
http://bitbucket.org/pypy/pypy/changeset/eb94727607d8/

Log:	kill pypy/tool/pytest/inttest.py and just use regular pytest objects
	for interp-level tests

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -121,9 +121,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -131,11 +128,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -164,7 +157,7 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py
deleted file mode 100644
--- a/pypy/tool/pytest/inttest.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Collects and executes interpreter-level tests.
-#
-# Most pypy tests are of this kind.
-
-import py
-from pypy.conftest import PyPyClassCollector
-
-
-class IntTestFunction(py.test.collect.Function):
-    pass
-
-
-class IntInstanceCollector(py.test.collect.Instance):
-    Function = IntTestFunction
-
-
-class IntClassCollector(PyPyClassCollector):
-    Instance = IntInstanceCollector
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -27,11 +27,11 @@
     result = testdir.runpytest("--collectonly")
     assert result.ret == 0
     result.stdout.fnmatch_lines([
-        "*IntTestFunction*test_func*",
-        "*IntClassCollector*TestClassInt*",
-        "*IntTestFunction*test_method*",
+        "*Function*test_func*",
+        "*Class*TestClassInt*",
+        "*Function*test_method*",
         "*AppClassCollector*AppTestClass*",
-        "*AppTestMethod*", 
+        "*AppTestMethod*",
     ])
 
 class TestSpaceConfig:
@@ -133,5 +133,5 @@
 
     x = 43
     info = raises(ZeroDivisionError, "x/0")
-    assert info.type is ZeroDivisionError    
-    assert isinstance(info.value, ZeroDivisionError)    
+    assert info.type is ZeroDivisionError
+    assert isinstance(info.value, ZeroDivisionError)

From pypy.commits at gmail.com  Tue Jun  7 18:46:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 07 Jun 2016 15:46:19 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: blind attempt to fix posix tests
Message-ID: <57574ebb.86981c0a.7317.ffff9e4e@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85016:cbdda232996a
Date: 2016-06-07 23:46 +0100
http://bitbucket.org/pypy/pypy/changeset/cbdda232996a/

Log:	blind attempt to fix posix tests

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1183,8 +1183,8 @@
             except UnicodeEncodeError:
                 py.test.skip("encoding not good enough")
         else:
-            cls.save_fs_encoding = space.sys.filesystemencoding
-            space.sys.filesystemencoding = "utf-8"
+            cls.save_fs_encoding = cls.space.sys.filesystemencoding
+            cls.space.sys.filesystemencoding = "utf-8"
 
     def teardown_class(cls):
         try:

From pypy.commits at gmail.com  Tue Jun  7 19:18:07 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Tue, 07 Jun 2016 16:18:07 -0700 (PDT)
Subject: [pypy-commit] pypy incminimark-ll_assert: Add assertions for an
 initialized nursery.
Message-ID: <5757562f.426dc20a.c03af.ffff96de@mx.google.com>

Author: Devin Jeanpierre 
Branch: incminimark-ll_assert
Changeset: r85017:103f0b255b8d
Date: 2016-06-07 14:22 -0700
http://bitbucket.org/pypy/pypy/changeset/103f0b255b8d/

Log:	Add assertions for an initialized nursery.

	This should make it easier to debug if you accidentally allocate
	during GC.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -643,6 +643,8 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
+            ll_assert(self.nursery_free != llmemory.NULL,
+                      "uninitialized nursery")
             result = self.nursery_free
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
@@ -702,6 +704,8 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
+            ll_assert(self.nursery_free != llmemory.NULL,
+                      "uninitialized nursery")
             result = self.nursery_free
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:

From pypy.commits at gmail.com  Tue Jun  7 19:18:09 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Tue, 07 Jun 2016 16:18:09 -0700 (PDT)
Subject: [pypy-commit] pypy incminimark-ll_assert: Convert assertions to
 ll_assert, add "NOT_RPYTHON" to remaining assert.
Message-ID: <57575631.665ec20a.79856.ffffe3e1@mx.google.com>

Author: Devin Jeanpierre 
Branch: incminimark-ll_assert
Changeset: r85018:b7b5f56b6c6e
Date: 2016-06-07 16:16 -0700
http://bitbucket.org/pypy/pypy/changeset/b7b5f56b6c6e/

Log:	Convert assertions to ll_assert, add "NOT_RPYTHON" to remaining
	assert.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -281,11 +281,12 @@
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
+        "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
-        
+
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
@@ -1143,7 +1144,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj),
+                  "Can't forward an object outside the nursery.")
         tid = self.header(obj).tid
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1467,7 +1469,8 @@
                 objhdr.tid |= GCFLAG_CARDS_SET
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1552,7 +1555,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1725,12 +1729,15 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(
+            self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "pinned_objects_in_nursery != surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(
+                cur >= prev, "pinned objects encountered in backwards order")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1788,7 +1795,8 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN,
+                  "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):

From pypy.commits at gmail.com  Tue Jun  7 23:11:32 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Tue, 07 Jun 2016 20:11:32 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: fix formatting
Message-ID: <57578ce4.c409c20a.8011e.3e31@mx.google.com>

Author: Philip Jenvey 
Branch: extradoc
Changeset: r754:d7eae3f0f84d
Date: 2016-06-07 20:11 -0700
http://bitbucket.org/pypy/pypy.org/changeset/d7eae3f0f84d/

Log:	fix formatting

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -413,7 +413,7 @@
 a1710ae6f15b567bf3c8fd608553ad48  pypy-5.1.0-win32.zip

pypy2.7-v5.3 md5:

-
+
 05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
 7d01e12eaca473258801faebc2db12d8  pypy2-v5.3.0-linux64.tar.bz2
 bf1640973865c5ca1bc88e299455cbcc  pypy2-v5.3.0-linux-armel.tar.bz2
@@ -423,7 +423,8 @@
 c39a578078ab3145d2a584cacf4c164c  pypy2-v5.3.0-s390x.tar.bz2
 45ce35a438ed8ae1539cc05407d43965  pypy2-v5.3.0-src.tar.bz2
 24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
-f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
+f6197adf58bfa32bcb18451289da1c7c pypy2-v5.3.0-win32.zip +

pypy3.3-v5.2-alpha md5:

 1176464541dff42e685bf8a9bb393796  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -495,8 +496,8 @@
 3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
 53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
-

pypy2.7-5.3 sha1

-
+

pypy2.7-5.3 sha1:

+
 401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
 939a49319ed8e25ecb9f646ba50f0618eb52c99b  pypy2-v5.3.0-linux64.tar.bz2
 00fe6e6d672c65d9096b45a708a1be95bca412c4  pypy2-v5.3.0-linux-armel.tar.bz2
@@ -506,9 +507,10 @@
 1d4f5f547d798294755fc9e14e2b6490e8a5d194  pypy2-v5.3.0-s390x.tar.bz2
 7bdb1cfc604192fc2a39023588d648a30975f0e4  pypy2-v5.3.0-src.tar.bz2
 18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
-076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
-

pypy2.7-5.3 sha256

-
+076251ba3b44435dc11867dab00f392b058bdc7c pypy2-v5.3.0-win32.zip + +

pypy2.7-5.3 sha256:

+
 bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
 ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588  pypy2-v5.3.0-linux64.tar.bz2
 81b6f589a947d7353bb69408c46d4833d6e9cb501f3c3f0c73bd28d0e3df69aa  pypy2-v5.3.0-linux-armel.tar.bz2
@@ -518,7 +520,8 @@
 1ccc0ce08dd55d188d3971331020a1c82e917e418846d2c2c07a225733d85b1e  pypy2-v5.3.0-s390x.tar.bz2
 4142eb8f403810bc88a4911792bb5a502e152df95806e33e69050c828cd160d5  pypy2-v5.3.0-src.tar.bz2
 09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
-32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
+32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20 pypy2-v5.3.0-win32.zip +

pypy3.3-v5.2-alpha sha1:

 03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -449,7 +449,7 @@
     204273a21dbf71c0827966265c40eb7a  pypy-5.1.0-src.zip
     a1710ae6f15b567bf3c8fd608553ad48  pypy-5.1.0-win32.zip
 
-pypy2.7-v5.3 md5:
+pypy2.7-v5.3 md5::
 
     05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
     7d01e12eaca473258801faebc2db12d8  pypy2-v5.3.0-linux64.tar.bz2
@@ -534,7 +534,7 @@
     3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
     53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
 
-pypy2.7-5.3 sha1
+pypy2.7-5.3 sha1::
 
     401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
     939a49319ed8e25ecb9f646ba50f0618eb52c99b  pypy2-v5.3.0-linux64.tar.bz2
@@ -547,7 +547,7 @@
     18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
     076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
 
-pypy2.7-5.3 sha256
+pypy2.7-5.3 sha256::
 
     bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
     ac336e8877ed676bf87a9a546d5926b6afc4679fa2d3fdf9f3ca56f28ec40588  pypy2-v5.3.0-linux64.tar.bz2

From pypy.commits at gmail.com  Wed Jun  8 03:01:43 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 08 Jun 2016 00:01:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix what is new on default
Message-ID: <5757c2d7.6a56c20a.f4dfe.4c4d@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r85019:2c8c9d7fb095
Date: 2016-06-08 08:53 +0200
http://bitbucket.org/pypy/pypy/changeset/2c8c9d7fb095/

Log:	fix what is new on default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -8,3 +8,14 @@
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -143,14 +143,3 @@
 Finish the work already partially merged in cpyext-for-merge. Adds support
 for ByteArrayObject using the nonmovable-list, which also enables
 buffer(bytearray()) 
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.

From pypy.commits at gmail.com  Wed Jun  8 03:01:45 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 08 Jun 2016 00:01:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: (s390x) fixed integer in test,
 should have been hexadecimal
Message-ID: <5757c2d9.2426c20a.4a6b9.514d@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r85020:b35e97b291d7
Date: 2016-06-08 09:00 +0200
http://bitbucket.org/pypy/pypy/changeset/b35e97b291d7/

Log:	(s390x) fixed integer in test, should have been hexadecimal

diff --git a/rpython/translator/platform/arch/test/test_s390x.py b/rpython/translator/platform/arch/test/test_s390x.py
--- a/rpython/translator/platform/arch/test/test_s390x.py
+++ b/rpython/translator/platform/arch/test/test_s390x.py
@@ -15,7 +15,7 @@
 processor 0: machine = 12345
 processor 1: version = FF, identification = AF
     """.splitlines())
-    assert ids == [(0, None, None, 12345),
+    assert ids == [(0, None, None, 0x12345),
                    (1, 'FF', 'AF', 0),
                   ]
 

From pypy.commits at gmail.com  Wed Jun  8 03:44:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 00:44:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: Change the registers used by
 'malloc_nursery' so that it won't force
Message-ID: <5757ccf0.0654c20a.c16a9.60b9@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85021:78ccb9fbb54f
Date: 2016-06-08 08:53 +0200
http://bitbucket.org/pypy/pypy/changeset/78ccb9fbb54f/

Log:	Change the registers used by 'malloc_nursery' so that it won't force
	away 'eax' returned by a likely previous 'call'.

	Add a before_call()-like equivalent before a few operations like
	'malloc_nursery', to move the unrelated values from the registers
	into other registers instead of to the stack.

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -901,6 +901,8 @@
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
         size = size_box.getint()
+        # hint: try to move unrelated registers away from r0 and r1 now
+        self.rm.spill_or_move_registers_before_call([r.r0, r.r1])
 
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
         t = TempInt()
@@ -924,6 +926,7 @@
         # sizeloc must be in a register, but we can free it now
         # (we take care explicitly of conflicts with r0 or r1)
         sizeloc = self.rm.make_sure_var_in_reg(size_box)
+        self.rm.spill_or_move_registers_before_call([r.r0, r.r1]) # sizeloc safe
         self.rm.possibly_free_var(size_box)
         #
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
@@ -951,6 +954,11 @@
         arraydescr = op.getdescr()
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
+        # can only use spill_or_move_registers_before_call() as a hint if
+        # we are sure that length_box stays alive and won't be overridden
+        # (it should always be the case, see below, but better safe than sorry)
+        if self.rm.stays_alive(length_box):
+            self.rm.spill_or_move_registers_before_call([r.r0, r.r1])
         # the result will be in r0
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
         # we need r1 as a temporary
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -579,11 +579,26 @@
         new_free_regs.append(self.reg_bindings.pop(v))
 
     def before_call(self, force_store=[], save_all_regs=0):
-        """Spill or move some registers before a call.  By default,
-        this means: for every register in 'self.save_around_call_regs',
+        self.spill_or_move_registers_before_call(self.save_around_call_regs,
+                                                 force_store, save_all_regs)
+
+    def spill_or_move_registers_before_call(self, save_sublist,
+                                            force_store=[], save_all_regs=0):
+        """Spill or move some registers before a call.
+
+        By default, this means: for every register in 'save_sublist',
         if there is a variable there and it survives longer than
         the current operation, then it is spilled/moved somewhere else.
 
+        WARNING: this might do the equivalent of possibly_free_vars()
+        on variables dying in the current operation.  It won't
+        immediately overwrite registers that used to be occupied by
+        these variables, though.  Use this function *after* you finished
+        calling self.loc() or self.make_sure_var_in_reg(), i.e. when you
+        know the location of all input arguments.  These locations stay
+        valid, but only *if they are in self.save_around_call_regs,*
+        not if they are callee-saved registers!
+
         'save_all_regs' can be 0 (default set of registers), 1 (do that
         for all registers), or 2 (default + gc ptrs).
 
@@ -612,6 +627,16 @@
         anyway, as a local hack in this function, because on x86 CPUs
         such register-register moves are almost free.
         """
+        if not we_are_translated():
+            # 'save_sublist' is either the whole
+            # 'self.save_around_call_regs', or a sublist thereof, and
+            # then only those registers are spilled/moved.  But when
+            # we move them, we never move them to other registers in
+            # 'self.save_around_call_regs', to avoid ping-pong effects
+            # where the same value is constantly moved around.
+            for reg in save_sublist:
+                assert reg in self.save_around_call_regs
+
         new_free_regs = []
         move_or_spill = []
 
@@ -631,7 +656,7 @@
                 # we need to spill all GC ptrs in this mode
                 self._bc_spill(v, new_free_regs)
                 #
-            elif reg not in self.save_around_call_regs:
+            elif reg not in save_sublist:
                 continue  # in a register like ebx/rbx: it is fine where it is
                 #
             else:
@@ -663,6 +688,7 @@
                 if not we_are_translated():
                     if move_or_spill:
                         assert max_age <= min([_a for _, _a in move_or_spill])
+                    assert reg in save_sublist
                     assert reg in self.save_around_call_regs
                     assert new_reg not in self.save_around_call_regs
                 self.assembler.regalloc_mov(reg, new_reg)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -204,20 +204,20 @@
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
-        The arguments are passed in eax and edi, as follows:
+        The arguments are passed in ecx and edx, as follows:
 
-        kind == 'fixed': nursery_head in eax and the size in edi - eax.
+        kind == 'fixed': nursery_head in ecx and the size in (edx - ecx).
 
-        kind == 'str/unicode': length of the string to allocate in edi.
+        kind == 'str/unicode': length of the string to allocate in edx.
 
-        kind == 'var': length to allocate in edi, tid in eax,
+        kind == 'var': length to allocate in edx, tid in ecx,
                        and itemsize in the stack 1 (position esp+WORD).
 
-        This function must preserve all registers apart from eax and edi.
+        This function must preserve all registers apart from ecx and edx.
         """
         assert kind in ['fixed', 'str', 'unicode', 'var']
         mc = codebuf.MachineCodeBlockWrapper()
-        self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats)
+        self._push_all_regs_to_frame(mc, [ecx, edx], self.cpu.supports_floats)
         # the caller already did push_gcmap(store=True)
         #
         if kind == 'fixed':
@@ -231,32 +231,32 @@
         mc.SUB_ri(esp.value, 16 - WORD)  # restore 16-byte alignment
         # magically, the above is enough on X86_32 to reserve 3 stack places
         if kind == 'fixed':
-            mc.SUB_rr(edi.value, eax.value) # compute the size we want
-            # the arg is already in edi
+            mc.SUB_rr(edx.value, ecx.value) # compute the size we want
             if IS_X86_32:
-                mc.MOV_sr(0, edi.value)
+                mc.MOV_sr(0, edx.value)     # store the length
                 if hasattr(self.cpu.gc_ll_descr, 'passes_frame'):
-                    mc.MOV_sr(WORD, ebp.value)
-            elif hasattr(self.cpu.gc_ll_descr, 'passes_frame'):
-                # for tests only
-                mc.MOV_rr(esi.value, ebp.value)
+                    mc.MOV_sr(WORD, ebp.value)        # for tests only
+            else:
+                mc.MOV_rr(edi.value, edx.value)   # length argument
+                if hasattr(self.cpu.gc_ll_descr, 'passes_frame'):
+                    mc.MOV_rr(esi.value, ebp.value)   # for tests only
         elif kind == 'str' or kind == 'unicode':
             if IS_X86_32:
                 # stack layout: [---][---][---][ret].. with 3 free stack places
-                mc.MOV_sr(0, edi.value)     # store the length
-            else:
-                pass                        # length already in edi
+                mc.MOV_sr(0, edx.value)     # store the length
+            elif IS_X86_64:
+                mc.MOV_rr(edi.value, edx.value)   # length argument
         else:
             if IS_X86_32:
                 # stack layout: [---][---][---][ret][gcmap][itemsize]...
-                mc.MOV_sr(WORD * 2, edi.value)  # store the length
-                mc.MOV_sr(WORD * 1, eax.value)  # store the tid
-                mc.MOV_rs(edi.value, WORD * 5)  # load the itemsize
-                mc.MOV_sr(WORD * 0, edi.value)  # store the itemsize
+                mc.MOV_sr(WORD * 2, edx.value)  # store the length
+                mc.MOV_sr(WORD * 1, ecx.value)  # store the tid
+                mc.MOV_rs(edx.value, WORD * 5)  # load the itemsize
+                mc.MOV_sr(WORD * 0, edx.value)  # store the itemsize
             else:
                 # stack layout: [---][ret][gcmap][itemsize]...
-                mc.MOV_rr(edx.value, edi.value) # length
-                mc.MOV_rr(esi.value, eax.value) # tid
+                # (already in edx)              # length
+                mc.MOV_rr(esi.value, ecx.value) # tid
                 mc.MOV_rs(edi.value, WORD * 3)  # load the itemsize
         self.set_extra_stack_depth(mc, 16)
         mc.CALL(imm(follow_jump(addr)))
@@ -267,10 +267,11 @@
         mc.TEST_rr(eax.value, eax.value)
         mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later
         jz_location = mc.get_relative_pos()
+        mc.MOV_rr(ecx.value, eax.value)
         #
         nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr()
-        self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats)
-        mc.MOV(edi, heap(nursery_free_adr))   # load this in EDI
+        self._pop_all_regs_from_frame(mc, [ecx, edx], self.cpu.supports_floats)
+        mc.MOV(edx, heap(nursery_free_adr))   # load this in EDX
         self.pop_gcmap(mc)   # push_gcmap(store=True) done by the caller
         mc.RET()
         #
@@ -2441,9 +2442,9 @@
 
     def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap):
         assert size & (WORD-1) == 0     # must be correctly aligned
-        self.mc.MOV(eax, heap(nursery_free_adr))
-        self.mc.LEA_rm(edi.value, (eax.value, size))
-        self.mc.CMP(edi, heap(nursery_top_adr))
+        self.mc.MOV(ecx, heap(nursery_free_adr))
+        self.mc.LEA_rm(edx.value, (ecx.value, size))
+        self.mc.CMP(edx, heap(nursery_top_adr))
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr = self.mc.get_relative_pos()
         # save the gcmap
@@ -2452,19 +2453,19 @@
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
-        self.mc.MOV(heap(nursery_free_adr), edi)
+        self.mc.MOV(heap(nursery_free_adr), edx)
 
     def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr,
                                   sizeloc, gcmap):
-        if sizeloc is eax:
-            self.mc.MOV(edi, sizeloc)
-            sizeloc = edi
-        self.mc.MOV(eax, heap(nursery_free_adr))
-        if sizeloc is edi:
-            self.mc.ADD_rr(edi.value, eax.value)
+        if sizeloc is ecx:
+            self.mc.MOV(edx, sizeloc)
+            sizeloc = edx
+        self.mc.MOV(ecx, heap(nursery_free_adr))
+        if sizeloc is edx:
+            self.mc.ADD_rr(edx.value, ecx.value)
         else:
-            self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, 0, 0))
-        self.mc.CMP(edi, heap(nursery_top_adr))
+            self.mc.LEA_ra(edx.value, (ecx.value, sizeloc.value, 0, 0))
+        self.mc.CMP(edx, heap(nursery_top_adr))
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr = self.mc.get_relative_pos()
         # save the gcmap
@@ -2473,7 +2474,7 @@
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
-        self.mc.MOV(heap(nursery_free_adr), edi)
+        self.mc.MOV(heap(nursery_free_adr), edx)
 
     def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr,
                             lengthloc, itemsize, maxlength, gcmap,
@@ -2482,39 +2483,39 @@
         assert isinstance(arraydescr, ArrayDescr)
 
         # lengthloc is the length of the array, which we must not modify!
-        assert lengthloc is not eax and lengthloc is not edi
+        assert lengthloc is not ecx and lengthloc is not edx
         if isinstance(lengthloc, RegLoc):
             varsizeloc = lengthloc
         else:
-            self.mc.MOV(edi, lengthloc)
-            varsizeloc = edi
+            self.mc.MOV(edx, lengthloc)
+            varsizeloc = edx
 
         self.mc.CMP(varsizeloc, imm(maxlength))
         self.mc.J_il8(rx86.Conditions['A'], 0) # patched later
         jmp_adr0 = self.mc.get_relative_pos()
 
-        self.mc.MOV(eax, heap(nursery_free_adr))
+        self.mc.MOV(ecx, heap(nursery_free_adr))
         if valid_addressing_size(itemsize):
             shift = get_scale(itemsize)
         else:
-            shift = self._imul_const_scaled(self.mc, edi.value,
+            shift = self._imul_const_scaled(self.mc, edx.value,
                                             varsizeloc.value, itemsize)
-            varsizeloc = edi
+            varsizeloc = edx
 
-        # now varsizeloc is a register != eax.  The size of
+        # now varsizeloc is a register != ecx.  The size of
         # the variable part of the array is (varsizeloc << shift)
         assert arraydescr.basesize >= self.gc_minimal_size_in_nursery
         constsize = arraydescr.basesize + self.gc_size_of_header
         force_realignment = (itemsize % WORD) != 0
         if force_realignment:
             constsize += WORD - 1
-        self.mc.LEA_ra(edi.value, (eax.value, varsizeloc.value, shift,
+        self.mc.LEA_ra(edx.value, (ecx.value, varsizeloc.value, shift,
                                    constsize))
         if force_realignment:
-            self.mc.AND_ri(edi.value, ~(WORD - 1))
-        # now edi contains the total size in bytes, rounded up to a multiple
+            self.mc.AND_ri(edx.value, ~(WORD - 1))
+        # now edx contains the total size in bytes, rounded up to a multiple
         # of WORD, plus nursery_free_adr
-        self.mc.CMP(edi, heap(nursery_top_adr))
+        self.mc.CMP(edx, heap(nursery_top_adr))
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr1 = self.mc.get_relative_pos()
         #
@@ -2525,8 +2526,8 @@
         self.push_gcmap(self.mc, gcmap, store=True)
         if kind == rewrite.FLAG_ARRAY:
             self.mc.MOV_si(WORD, itemsize)
-            self.mc.MOV(edi, lengthloc)
-            self.mc.MOV_ri(eax.value, arraydescr.tid)
+            self.mc.MOV(edx, lengthloc)
+            self.mc.MOV_ri(ecx.value, arraydescr.tid)
             addr = self.malloc_slowpath_varsize
         else:
             if kind == rewrite.FLAG_STR:
@@ -2534,7 +2535,7 @@
             else:
                 assert kind == rewrite.FLAG_UNICODE
                 addr = self.malloc_slowpath_unicode
-            self.mc.MOV(edi, lengthloc)
+            self.mc.MOV(edx, lengthloc)
         self.mc.CALL(imm(follow_jump(addr)))
         self.mc.JMP_l8(0)      # jump to done, patched later
         jmp_location = self.mc.get_relative_pos()
@@ -2544,9 +2545,9 @@
         self.mc.overwrite(jmp_adr1-1, chr(offset))
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
-        self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
+        self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
-        self.mc.MOV(heap(nursery_free_adr), edi)
+        self.mc.MOV(heap(nursery_free_adr), edx)
         #
         offset = self.mc.get_relative_pos() - jmp_location
         assert 0 < offset <= 127
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -952,14 +952,16 @@
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
         size = size_box.getint()
-        # looking at the result
-        self.rm.force_allocate_reg(op, selected_reg=eax)
+        # hint: try to move unrelated registers away from eax and edx now
+        self.rm.spill_or_move_registers_before_call([ecx, edx])
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
         #
-        # We need edi as a temporary, but otherwise don't save any more
+        # We need edx as a temporary, but otherwise don't save any more
         # register.  See comments in _build_malloc_slowpath().
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
@@ -972,15 +974,16 @@
         size_box = op.getarg(0)
         assert not isinstance(size_box, Const) # we cannot have a const here!
         # sizeloc must be in a register, but we can free it now
-        # (we take care explicitly of conflicts with eax or edi)
+        # (we take care explicitly of conflicts with ecx or edx)
         sizeloc = self.rm.make_sure_var_in_reg(size_box)
+        self.rm.spill_or_move_registers_before_call([ecx, edx])  # sizeloc safe
         self.rm.possibly_free_var(size_box)
-        # the result will be in eax
-        self.rm.force_allocate_reg(op, selected_reg=eax)
-        # we need edi as a temporary
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
+        # we need edx as a temporary
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
@@ -997,16 +1000,21 @@
         arraydescr = op.getdescr()
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
-        # the result will be in eax
-        self.rm.force_allocate_reg(op, selected_reg=eax)
-        # we need edi as a temporary
+        # can only use spill_or_move_registers_before_call() as a hint if
+        # we are sure that length_box stays alive and won't be overridden
+        # (it should always be the case, see below, but better safe than sorry)
+        if self.rm.stays_alive(length_box):
+            self.rm.spill_or_move_registers_before_call([ecx, edx])
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
+        # we need edx as a temporary
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         # length_box always survives: it's typically also present in the
         # next operation that will copy it inside the new array.  It's
-        # fine to load it from the stack too, as long as it's != eax, edi.
+        # fine to load it from the stack too, as long as it is != ecx, edx.
         lengthloc = self.rm.loc(length_box)
         self.rm.possibly_free_var(length_box)
         #
@@ -1225,6 +1233,8 @@
             raise AssertionError("bad unicode item size")
 
     def _consider_math_read_timestamp(self, op):
+        # hint: try to move unrelated registers away from eax and edx now
+        self.rm.spill_or_move_registers_before_call([eax, edx])
         tmpbox_high = TempVar()
         self.rm.force_allocate_reg(tmpbox_high, selected_reg=eax)
         if longlong.is_64_bit:

From pypy.commits at gmail.com  Wed Jun  8 03:44:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 00:44:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix comment
Message-ID: <5757ccf2.832c1c0a.20d42.2028@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85022:706f6ce2efe4
Date: 2016-06-08 09:17 +0200
http://bitbucket.org/pypy/pypy/changeset/706f6ce2efe4/

Log:	Fix comment

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -955,7 +955,7 @@
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
         # can only use spill_or_move_registers_before_call() as a hint if
-        # we are sure that length_box stays alive and won't be overridden
+        # we are sure that length_box stays alive and won't be freed now
         # (it should always be the case, see below, but better safe than sorry)
         if self.rm.stays_alive(length_box):
             self.rm.spill_or_move_registers_before_call([r.r0, r.r1])
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1001,7 +1001,7 @@
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
         # can only use spill_or_move_registers_before_call() as a hint if
-        # we are sure that length_box stays alive and won't be overridden
+        # we are sure that length_box stays alive and won't be freed now
         # (it should always be the case, see below, but better safe than sorry)
         if self.rm.stays_alive(length_box):
             self.rm.spill_or_move_registers_before_call([ecx, edx])

From pypy.commits at gmail.com  Wed Jun  8 03:44:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 00:44:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5757ccf4.c6e41c0a.eca97.2ef8@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85023:2384df43b1dd
Date: 2016-06-08 09:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2384df43b1dd/

Log:	merge heads

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -164,13 +164,6 @@
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
 
 class PyPyClassCollector(py.test.collect.Class):
     # All pypy Test classes have a "space" member.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,3 +5,17 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -205,7 +205,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    map = {"'" : (singleDFA, states_singleDFA),
+           '"' : (doubleDFA, states_doubleDFA),
            "r" : None,
            "R" : None,
            "u" : None,
@@ -257,25 +265,30 @@
     for uniPrefix in ("", "u", "U", "b", "B", ):
         for rawPrefix in ("", "r", "R"):
             prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
+            map[prefix + "'''"] = (single3DFA, states_single3DFA)
+            map[prefix + '"""'] = (double3DFA, states_double3DFA)
     return map
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append('\n')
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -298,22 +311,28 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap["\""]
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.genpytokenize import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -1,5 +1,5 @@
-import pytest
 from rpython.tool import udir
+from pypy.conftest import option
 from pypy.interpreter.gateway import interp2app
 
 def check_no_w_locals(space, w_frame):
@@ -11,7 +11,7 @@
         space = cls.space
         cls.w_udir = cls.space.wrap(str(udir.udir))
         cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1')))
-        if not pytest.config.option.runappdirect:
+        if not option.runappdirect:
             w_call_further = cls.space.appexec([], """():
                 def call_further(f):
                     return f()
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -1,4 +1,5 @@
 import py
+from pypy import conftest
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py
--- a/pypy/module/_continuation/test/test_translated.py
+++ b/pypy/module/_continuation/test/test_translated.py
@@ -89,7 +89,8 @@
 class AppTestWrapper:
     def setup_class(cls):
         "Run test_various_depths() when we are run with 'pypy py.test -A'."
-        if not py.test.config.option.runappdirect:
+        from pypy.conftest import option
+        if not option.runappdirect:
             py.test.skip("meant only for -A run")
 
 def _setup():
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -516,11 +516,12 @@
             assert s == 'bar\n'
 
 def test_flush_at_exit():
+    from pypy import conftest
     from pypy.tool.option import make_config, make_objspace
     from rpython.tool.udir import udir
 
     tmpfile = udir.join('test_flush_at_exit')
-    config = make_config(py.test.config.option)
+    config = make_config(conftest.option)
     space = make_objspace(config)
     space.appexec([space.wrap(str(tmpfile))], """(tmpfile):
         f = open(tmpfile, 'w')
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -222,10 +222,14 @@
         assert not closed[0]  # flush() called before file closed
         os.close(fd)
 
-def test_flush_at_exit(space):
+def test_flush_at_exit():
+    from pypy import conftest
+    from pypy.tool.option import make_config, make_objspace
     from rpython.tool.udir import udir
 
     tmpfile = udir.join('test_flush_at_exit')
+    config = make_config(conftest.option)
+    space = make_objspace(config)
     space.appexec([space.wrap(str(tmpfile))], """(tmpfile):
         import io
         f = io.open(tmpfile, 'w', encoding='ascii')
@@ -237,7 +241,12 @@
     assert tmpfile.read() == '42'
 
 
-def test_flush_at_exit_IOError_and_ValueError(space):
+def test_flush_at_exit_IOError_and_ValueError():
+    from pypy import conftest
+    from pypy.tool.option import make_config, make_objspace
+
+    config = make_config(conftest.option)
+    space = make_objspace(config)
     space.appexec([], """():
         import io
         class MyStream(io.IOBase):
diff --git a/pypy/module/_rawffi/test/test_tracker.py b/pypy/module/_rawffi/test/test_tracker.py
--- a/pypy/module/_rawffi/test/test_tracker.py
+++ b/pypy/module/_rawffi/test/test_tracker.py
@@ -1,5 +1,5 @@
 import py
-from pytest import option
+from pypy.conftest import option
 from pypy.module._rawffi.tracker import Tracker
 
 
@@ -44,3 +44,4 @@
 
     def teardown_class(cls):
         Tracker.DO_TRACING = False
+
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer,
+    PyVarObject, Py_buffer, size_t,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -14,14 +14,14 @@
 import pypy.module.__builtin__.operation as operation
 
 
- at cpython_api([Py_ssize_t], rffi.VOIDP)
+ at cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
     return lltype.malloc(rffi.VOIDP.TO, size,
                          flavor='raw',
                          add_memory_pressure=True)
 
- at cpython_api([rffi.VOIDP, Py_ssize_t], rffi.VOIDP)
+ at cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
         return lltype.malloc(rffi.VOIDP.TO, size,
diff --git a/pypy/module/gc/test/test_referents.py b/pypy/module/gc/test/test_referents.py
--- a/pypy/module/gc/test/test_referents.py
+++ b/pypy/module/gc/test/test_referents.py
@@ -1,4 +1,4 @@
-from pytest import option
+from pypy.conftest import option
 
 
 class AppTestReferents(object):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -13,7 +13,7 @@
 
 from pypy.module.imp import importing
 
-from pytest import config
+from pypy import conftest
 
 def setuppkg(pkgname, **entries):
     p = udir.join('impsubdir')
@@ -106,7 +106,7 @@
 
     # create compiled/x.py and a corresponding pyc file
     p = setuppkg("compiled", x = "x = 84")
-    if config.option.runappdirect:
+    if conftest.option.runappdirect:
         import marshal, stat, struct, os, imp
         code = py.code.Source(p.join("x.py").read()).compile()
         s3 = marshal.dumps(code)
@@ -168,7 +168,7 @@
     }
 
     def setup_class(cls):
-        cls.w_runappdirect = cls.space.wrap(config.option.runappdirect)
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.saved_modules = _setup(cls.space)
         #XXX Compile class
 
@@ -1494,6 +1494,8 @@
     spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
+        #if not conftest.option.runappdirect:
+        #    py.test.skip("meant as an -A test")
         tmpfile = udir.join('test_multithreaded_imp.py')
         tmpfile.write('''if 1:
             x = 666
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,4 +1,4 @@
-from pytest import config
+from pypy.conftest import option
 from pypy.module.micronumpy import constants as NPY
 
 
@@ -7,7 +7,7 @@
 
     @classmethod
     def setup_class(cls):
-        if config.option.runappdirect:
+        if option.runappdirect:
             import sys
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -2,6 +2,7 @@
 
 import sys
 
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
@@ -22,13 +23,13 @@
     # special values testing
     if isnan(a):
         if isnan(b):
-            return True, ''
+            return True,''
         raise AssertionError(msg + '%r should be nan' % (b,))
 
     if isinf(a):
         if a == b:
-            return True, ''
-        raise AssertionError(msg + 'finite result where infinity expected: '
+            return True,''
+        raise AssertionError(msg + 'finite result where infinity expected: '+ \
                           'expected %r, got %r' % (a, b))
 
     # if both a and b are zero, check whether they have the same sign
@@ -38,7 +39,7 @@
     if not a and not b:
         # only check it if we are running on top of CPython >= 2.6
         if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b):
-            raise AssertionError(msg +
+            raise AssertionError( msg + \
                     'zero has wrong sign: expected %r, got %r' % (a, b))
 
     # if a-b overflows, or b is infinite, return False.  Again, in
@@ -95,6 +96,7 @@
         cls.w_testcases128 = cls.space.wrap(list(parse_testfile(fname128)))
         cls.w_testcases64 = cls.space.wrap(list(parse_testfile(fname64)))
 
+        cls.w_runAppDirect = cls.space.wrap(option.runappdirect)
         cls.w_isWindows = cls.space.wrap(os.name == 'nt')
 
         if cls.runappdirect:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1,11 +1,11 @@
-from pytest import config
+from pypy.conftest import option
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 from pypy.interpreter.gateway import interp2app
 
 class BaseAppTestDtypes(BaseNumpyAppTest):
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)
-        if config.option.runappdirect:
+        if option.runappdirect:
             import platform
             bits, linkage = platform.architecture()
             ptr_size = int(bits[:-3]) // 8
@@ -1088,7 +1088,7 @@
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)
-        if config.option.runappdirect:
+        if option.runappdirect:
             cls.w_test_for_core_internal = cls.space.wrap(True)
         else:
             cls.w_test_for_core_internal = cls.space.wrap(False)
@@ -1515,7 +1515,7 @@
             else:
                 assert stor2[1] == '\x01'
                 assert stor2[0] == '\x00'
-        if config.option.runappdirect:
+        if option.runappdirect:
             cls.w_check_non_native = lambda *args : None
         else:
             cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2,7 +2,7 @@
 import py
 import sys
 
-from pytest import config
+from pypy.conftest import option
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.strides import Chunk, new_view, EllipsisChunk
 from pypy.module.micronumpy.ndarray import W_NDimArray
@@ -3850,7 +3850,7 @@
 
 class AppTestRepr(BaseNumpyAppTest):
     def setup_class(cls):
-        if config.option.runappdirect:
+        if option.runappdirect:
             py.test.skip("Can't be run directly.")
         BaseNumpyAppTest.setup_class.im_func(cls)
         cache = get_appbridge_cache(cls.space)
@@ -3867,7 +3867,7 @@
         assert repr(array(1.5).real) == "array(1.5)"
 
     def teardown_class(cls):
-        if config.option.runappdirect:
+        if option.runappdirect:
             return
         cache = get_appbridge_cache(cls.space)
         cache.w_array_repr = cls.old_array_repr
@@ -4343,7 +4343,7 @@
 
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
-        if config.option.runappdirect and '__pypy__' not in sys.builtin_module_names:
+        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
             py.test.skip("pypy only test")
         BaseNumpyAppTest.setup_class.im_func(cls)
 
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -1,9 +1,14 @@
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.conftest import option
 
 
 class AppTestObjectDtypes(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
 
+    def setup_class(cls):
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_runappdirect = cls.space.wrap(option.runappdirect)
+
     def test_scalar_from_object(self):
         from numpy import array
         import sys
diff --git a/pypy/module/micronumpy/test/test_support_app.py b/pypy/module/micronumpy/test/test_support_app.py
--- a/pypy/module/micronumpy/test/test_support_app.py
+++ b/pypy/module/micronumpy/test/test_support_app.py
@@ -3,12 +3,11 @@
 import py
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
-from pytest import config
+from pypy.conftest import option
 
 class AppTestSupport(BaseNumpyAppTest):
     def setup_class(cls):
-        if (config.option.runappdirect and
-                '__pypy__' not in sys.builtin_module_names):
+        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
             py.test.skip("pypy only test")
         BaseNumpyAppTest.setup_class.im_func(cls)
 
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -20,6 +20,10 @@
         self.w_sockets = self.space.wrap([])
         if platform.machine().startswith('arm'):
             self.w_timeout = self.space.wrap(0.06)
+        if platform.machine().startswith('s390x'):
+            # s390x is not slow, but it seems there is one case when epoll
+            # modify method is called that takes longer on s390x
+            self.w_timeout = self.space.wrap(0.06)
         else:
             self.w_timeout = self.space.wrap(0.02)
 
diff --git a/pypy/module/test_lib_pypy/support.py b/pypy/module/test_lib_pypy/support.py
--- a/pypy/module/test_lib_pypy/support.py
+++ b/pypy/module/test_lib_pypy/support.py
@@ -1,6 +1,6 @@
 import py
 
-from pytest import config
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
 
 def import_lib_pypy(space, name, skipmsg=None):
@@ -9,7 +9,7 @@
 
     Raises a pytest Skip on ImportError if a skip message was specified.
     """
-    if config.option.runappdirect:
+    if option.runappdirect:
         try:
             mod = __import__('lib_pypy.' + name)
         except ImportError as e:
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,9 +1,9 @@
-from pytest import config
+from pypy import conftest
 
 
 class AppTestBytesArray:
     def setup_class(cls):
-        cls.w_runappdirect = cls.space.wrap(config.option.runappdirect)
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
 
     def test_basics(self):
         b = bytearray()
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -1,4 +1,5 @@
-from pytest import config
+from __future__ import with_statement
+from pypy.conftest import option
 
 class AppTestObject:
 
@@ -6,13 +7,13 @@
         from pypy.interpreter import gateway
         import sys
 
-        cpython_behavior = (not config.option.runappdirect
+        cpython_behavior = (not option.runappdirect
                             or not hasattr(sys, 'pypy_translation_info'))
 
         space = cls.space
         cls.w_cpython_behavior = space.wrap(cpython_behavior)
         cls.w_cpython_version = space.wrap(tuple(sys.version_info))
-        cls.w_appdirect = space.wrap(config.option.runappdirect)
+        cls.w_appdirect = space.wrap(option.runappdirect)
         cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info'))
 
         def w_unwrap_wrap_unicode(space, w_obj):
diff --git a/pypy/objspace/std/test/test_proxy_usercreated.py b/pypy/objspace/std/test/test_proxy_usercreated.py
--- a/pypy/objspace/std/test/test_proxy_usercreated.py
+++ b/pypy/objspace/std/test/test_proxy_usercreated.py
@@ -5,7 +5,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
 from pypy.objspace.std.transparent import register_proxyable
-from pytest import config
+from pypy.conftest import option
 
 
 class W_Wrapped(W_Root):
@@ -25,7 +25,7 @@
 
 class AppTestProxyNewtype(AppProxy):
     def setup_class(cls):
-        if config.option.runappdirect:
+        if option.runappdirect:
             py.test.skip("Impossible to run on appdirect")
         AppProxy.setup_class.im_func(cls)
         cls.w_wrapped = cls.space.wrap(W_Wrapped())
diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py
--- a/pypy/objspace/test/test_binop_overriding.py
+++ b/pypy/objspace/test/test_binop_overriding.py
@@ -1,5 +1,5 @@
 # test about the binop operation rule, see issue 412
-from pytest import config
+from pypy.conftest import option
 
 class AppTestBinopCombinations:
 
@@ -83,7 +83,7 @@
     return Base, do_test
 """)
         cls.w_helpers = w_helpers
-        cls.w_appdirect = cls.space.wrap(config.option.runappdirect)
+        cls.w_appdirect = cls.space.wrap(option.runappdirect)
 
     def test_overriding_base_binop_explict(self):
         class MulBase(object):
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -184,6 +184,7 @@
                 source = str(source).strip()
             except py.error.ENOENT:
                 source = None
+            from pypy import conftest
             if source and py.test.config._assertstate.mode != "off":
                 msg = interpret(source, runner, should_fail=True)
                 space.setattr(w_self, space.wrap('args'),
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -3,13 +3,14 @@
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_config, make_objspace
 from pypy.tool.pytest import appsupport
+from pypy.conftest import option
 
 _SPACECACHE={}
 def gettestobjspace(**kwds):
     """ helper for instantiating and caching space's for testing.
     """
     try:
-        config = make_config(py.test.config.option,**kwds)
+        config = make_config(option,**kwds)
     except ConflictConfigError as e:
         # this exception is typically only raised if a module is not available.
         # in this case the test should be skipped
@@ -18,7 +19,7 @@
     try:
         return _SPACECACHE[key]
     except KeyError:
-        if getattr(py.test.config.option, 'runappdirect', None):
+        if getattr(option, 'runappdirect', None):
             return TinyObjSpace(**kwds)
         space = maketestobjspace(config)
         _SPACECACHE[key] = space
@@ -26,7 +27,7 @@
 
 def maketestobjspace(config=None):
     if config is None:
-        config = make_config(py.test.config.option)
+        config = make_config(option)
     if config.objspace.usemodules.thread:
         config.translation.thread = True
     space = make_objspace(config)
diff --git a/rpython/doc/arch/index.rst b/rpython/doc/arch/index.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/arch/index.rst
@@ -0,0 +1,11 @@
+.. _arch_index:
+
+Architecture specific notes
+===========================
+
+Here you can find some architecture specific notes.
+
+.. toctree::
+    :maxdepth: 1
+
+    s390x
diff --git a/rpython/doc/arch/s390x.rst b/rpython/doc/arch/s390x.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/arch/s390x.rst
@@ -0,0 +1,34 @@
+.. _s390x:
+
+IBM Mainframe S390X
+===================
+
+Our JIT implements the 64 bit version of the IBM Mainframe called s390x.
+Note that this architecture is big endian.
+
+Currently supported ISAs:
+
+* z13 (released January 2015)
+* zEC12 (released September 2012)
+* z196 (released August 2010)
+* z10 (released February 2008)
+
+To check if all the necessary CPU facilities are installed
+on the subject machine, please run the test using a copy of the pypy
+source code::
+
+    $ ./pytest.py rpython/jit/backend/zarch/test/test_assembler -v -k 'test_facility'
+
+In addition you can run the auto encoding test to check if your Linux GCC tool chain
+is able to compile all instructions used in the JIT backend::
+
+    $ ./pytest.py rpython/jit/backend/zarch/test/test_auto_encoding.py -v
+
+Translating
+-----------
+
+Specifically check for these two dependencies. On old versions of some
+Linux distributions ship older versions.
+
+* libffi (version should do > 3.0.+).
+* CPython 2.7.+.
diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -37,7 +37,6 @@
 
    arm
    logging
-   s390x
 
 
 Writing your own interpreter in RPython
@@ -61,6 +60,7 @@
    getting-started
    dir-reference
    jit/index
+   arch/index
    translation
    rtyper
    garbage_collection
diff --git a/rpython/doc/s390x.rst b/rpython/doc/s390x.rst
deleted file mode 100644
--- a/rpython/doc/s390x.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _s390x:
-
-S390X JIT Backend
-=================
-
-Our JIT implements the 64 bit version of the IBM Mainframe called s390x.
-Note that this architecture is big endian.
-
-The following facilities need to be installed to operate
-correctly (all of the machines used for development these where installed):
-
-* General-Instructions-Extension
-* Long-Displacement
-* Binary Floating Point (IEEE)
-
-Translating
------------
-
-Ensure that libffi is installed (version should do > 3.0.+).
-CPython should be version 2.7.+.
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -439,7 +439,7 @@
     prepare_int_lshift = helper.prepare_binary_op
     prepare_int_rshift = helper.prepare_binary_op
     prepare_uint_rshift = helper.prepare_binary_op
-    prepare_uint_mul_high = helper.prepare_binary_op
+    prepare_uint_mul_high = helper.prepare_int_mul_ovf
 
     prepare_int_add_ovf = helper.prepare_binary_op
     prepare_int_sub_ovf = helper.prepare_binary_op
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -29,6 +29,7 @@
     'MGHI':    ('ri',    ['\xA7','\x0D']),
     'MSGFI':   ('ril',   ['\xC2','\x00']),
     'MLGR':    ('rre',   ['\xB9','\x86'], 'eo,r'),
+    'MLG':     ('rxy',   ['\xE3','\x86'], 'eo,bid'),
     # div/mod
     'DSGR':    ('rre',   ['\xB9','\x0D'], 'eo,r'),
     'DSG':     ('rxy',   ['\xE3','\x0D'], 'eo,bidl'),
@@ -44,7 +45,6 @@
 
     # rotating
     'RISBG':   ('rie_f',   ['\xEC','\x55']),
-    'RISBGN':  ('rie_f',   ['\xEC','\x59']),
 
     # invert & negative & absolute
     'LPGR':    ('rre',   ['\xB9','\x00']),
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -160,11 +160,15 @@
         omc.BRC(c.ANY, l.imm(label_end - jmp_neither_lqlr_overflow))
         omc.overwrite()
 
-    emit_int_floordiv = gen_emit_div_mod('DSGR', 'DSG')
-    emit_uint_floordiv = gen_emit_div_mod('DLGR', 'DLG')
-    # NOTE division sets one register with the modulo value, thus
-    # the regalloc ensures the right register survives.
-    emit_int_mod = gen_emit_div_mod('DSGR', 'DSG')
+    def emit_uint_mul_high(self, op, arglocs, regalloc):
+        r0, _, a1 = arglocs
+        # _ carries the value, contents of r0 are ignored
+        assert not r0.is_imm()
+        assert not a1.is_imm()
+        if a1.is_core_reg():
+            self.mc.MLGR(r0, a1)
+        else:
+            self.mc.MLG(r0, a1)
 
     def emit_int_invert(self, op, arglocs, regalloc):
         l0, = arglocs
diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py
--- a/rpython/jit/backend/zarch/regalloc.py
+++ b/rpython/jit/backend/zarch/regalloc.py
@@ -733,9 +733,6 @@
     prepare_int_sub_ovf = helper.prepare_int_sub
     prepare_int_mul = helper.prepare_int_mul
     prepare_int_mul_ovf = helper.prepare_int_mul_ovf
-    prepare_int_floordiv = helper.prepare_int_div
-    prepare_uint_floordiv = helper.prepare_int_div
-    prepare_int_mod = helper.prepare_int_mod
     prepare_nursery_ptr_increment = prepare_int_add
 
     prepare_int_and = helper.prepare_int_logic
@@ -746,6 +743,18 @@
     prepare_int_lshift  = helper.prepare_int_shift
     prepare_uint_rshift = helper.prepare_int_shift
 
+    def prepare_uint_mul_high(self, op):
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        if a0.is_constant():
+            a0, a1 = a1, a0
+        if helper.check_imm32(a1):
+            l1 = self.ensure_reg(a1)
+        else:
+            l1 = self.ensure_reg_or_pool(a1)
+        lr,lq = self.rm.ensure_even_odd_pair(a0, op, bind_first=True)
+        return [lr, lq, l1]
+
     prepare_int_le = helper.generate_cmp_op()
     prepare_int_lt = helper.generate_cmp_op()
     prepare_int_ge = helper.generate_cmp_op()
diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py
--- a/rpython/jit/backend/zarch/test/test_assembler.py
+++ b/rpython/jit/backend/zarch/test/test_assembler.py
@@ -155,7 +155,15 @@
         s64 = bin(fac_data[1])[2:]
         print(f64)
         print(s64)
+        for i,c in enumerate(f64):
+            print('index: %d is set? %s' % (i,c))
+
+        assert f64[1] == '1' # The z/Architecture architectural mode is installed.
+        assert f64[2] == '1' # The z/Architecture architectural mode is active.
         assert f64[18] == '1' # long displacement facility
+        assert f64[21] == '1' # extended immediate facility
+        assert f64[34] == '1' # general instruction facility
+        assert f64[41] == '1' # floating-point-support-enhancement
 
     def test_load_byte_zero_extend(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
@@ -189,7 +197,7 @@
     @py.test.mark.parametrize('p', [2**32,2**32+1,2**63-1,2**63-2,0,1,2,3,4,5,6,7,8,10001])
     def test_align_withroll(self, p):
         self.a.mc.load_imm(r.r2, p & 0xffffFFFFffffFFFF)
-        self.a.mc.RISBGN(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0))
+        self.a.mc.RISBG(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0))
         self.a.mc.BCR(con.ANY, r.r14)
         assert run_asm(self.a) == rffi.cast(rffi.ULONG,p) & ~(7)
 
@@ -214,7 +222,7 @@
         n = 13
         l = loc
         self.a.mc.load_imm(r.r2, 7<

Author: Armin Rigo 
Branch: 
Changeset: r85024:b98430da0285
Date: 2016-06-08 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/b98430da0285/

Log:	Remove this file, exact copy of test_rzpy_vmprof.py

diff --git a/rpython/jit/backend/x86/test/test_zvmprof.py b/rpython/jit/backend/x86/test/test_zvmprof.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_zvmprof.py
+++ /dev/null
@@ -1,7 +0,0 @@
-
-from rpython.jit.backend.llsupport.test.zrpy_vmprof_test import CompiledVmprofTest
-
-class TestZVMprof(CompiledVmprofTest):
-    
-    gcrootfinder = "shadowstack"
-    gc = "incminimark"
\ No newline at end of file

From pypy.commits at gmail.com  Wed Jun  8 04:31:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 01:31:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: Comment
Message-ID: <5757d7f3.0e9e1c0a.42103.3a14@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85025:29c6a8a14464
Date: 2016-06-08 10:32 +0200
http://bitbucket.org/pypy/pypy/changeset/29c6a8a14464/

Log:	Comment

diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -201,6 +201,9 @@
         else:
             copy_string_to_raw(llstr(src_string), dest_data, 0, n)
     else:
+        # nowadays this case should be rare or impossible: as far as
+        # I know, all common types implementing the *writable* buffer
+        # interface now support get_raw_address()
         if src_is_ptr:
             for i in range(n):
                 dest_buf.setitem(i, src_data[i])

From pypy.commits at gmail.com  Wed Jun  8 04:49:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 08 Jun 2016 01:49:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: printing random seed in test
 (test_runner), 
Message-ID: <5757dc36.86981c0a.7317.405f@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r85026:8d195daa0502
Date: 2016-06-08 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/8d195daa0502/

Log:	printing random seed in test (test_runner), more tightly packing the
	stack when calling with release gil (removes 2 WORDs)

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2825,6 +2825,7 @@
         from rpython.rlib.rarithmetic import r_singlefloat
         from rpython.translator.c import primitive
 
+
         def same_as_for_box(b):
             if b.type == 'i':
                 return rop.SAME_AS_I
@@ -2835,6 +2836,8 @@
 
         cpu = self.cpu
         rnd = random.Random(525)
+        seed = py.test.config.option.randomseed
+        print("random seed %d" % seed)
 
         ALL_TYPES = [
             (types.ulong,  lltype.Unsigned),
diff --git a/rpython/jit/backend/zarch/callbuilder.py b/rpython/jit/backend/zarch/callbuilder.py
--- a/rpython/jit/backend/zarch/callbuilder.py
+++ b/rpython/jit/backend/zarch/callbuilder.py
@@ -12,6 +12,8 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.jit.backend.llsupport.descr import CallDescr
 
+CALL_RELEASE_GIL_STACK_OFF = 6*WORD
+
 class CallBuilder(AbstractCallBuilder):
     GPR_ARGS = [r.r2, r.r3, r.r4, r.r5, r.r6]
     FPR_ARGS =  [r.f0, r.f2, r.f4, r.f6]
@@ -85,8 +87,8 @@
         self.subtracted_to_sp += len(stack_params) * WORD
         base = len(stack_params) * WORD
         if self.is_call_release_gil:
-            self.subtracted_to_sp += 8*WORD
-            base += 8*WORD
+            self.subtracted_to_sp += CALL_RELEASE_GIL_STACK_OFF
+            base += CALL_RELEASE_GIL_STACK_OFF
         for idx,i in enumerate(stack_params):
             loc = arglocs[i]
             offset = STD_FRAME_SIZE_IN_BYTES - base + 8 * idx
@@ -187,7 +189,7 @@
         RSHADOWPTR = self.RSHADOWPTR
         RFASTGILPTR = self.RFASTGILPTR
         #
-        pos = STD_FRAME_SIZE_IN_BYTES - 7*WORD
+        pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF
         self.mc.STMG(r.r8, r.r13, l.addr(pos, r.SP))
         #
         # Save this thread's shadowstack pointer into r8, for later comparison
@@ -286,7 +288,7 @@
         if gcrootmap:
             if gcrootmap.is_shadow_stack and self.is_call_release_gil:
                 self.mc.LGR(r.SCRATCH, RSHADOWOLD)
-        pos = STD_FRAME_SIZE_IN_BYTES - 7*WORD
+        pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF
         self.mc.LMG(r.r8, r.r13, l.addr(pos, r.SP))
 
     def write_real_errno(self, save_err):

From pypy.commits at gmail.com  Wed Jun  8 06:31:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 08 Jun 2016 03:31:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: reduced diff to cpython's Python.asdl
 (MatMul -> MatMult)
Message-ID: <5757f3e6.2450c20a.6ab27.3b91@mx.google.com>

Author: Richard Plangger 
Branch: py3.5
Changeset: r85027:f03edd99abf9
Date: 2016-06-08 12:30 +0200
http://bitbucket.org/pypy/pypy/changeset/f03edd99abf9/

Log:	reduced diff to cpython's Python.asdl (MatMul -> MatMult) copied
	over asdl.py from cpython (3.5)

diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -91,8 +91,8 @@
 
     boolop = And | Or
 
-    operator = Add | Sub | Mult | Div | Mod | Pow | LShift 
-                 | RShift | BitOr | BitXor | BitAnd | FloorDiv | MatMul
+    operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift
+                 | RShift | BitOr | BitXor | BitAnd | FloorDiv
 
     unaryop = Invert | Not | UAdd | USub
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl.py b/pypy/interpreter/astcompiler/tools/asdl.py
--- a/pypy/interpreter/astcompiler/tools/asdl.py
+++ b/pypy/interpreter/astcompiler/tools/asdl.py
@@ -1,235 +1,52 @@
-"""An implementation of the Zephyr Abstract Syntax Definition Language.
+#-------------------------------------------------------------------------------
+# Parser for ASDL [1] definition files. Reads in an ASDL description and parses
+# it into an AST that describes it.
+#
+# The EBNF we're parsing here: Figure 1 of the paper [1]. Extended to support
+# modules and attributes after a product. Words starting with Capital letters
+# are terminals. Literal tokens are in "double quotes". Others are
+# non-terminals. Id is either TokenId or ConstructorId.
+#
+# module        ::= "module" Id "{" [definitions] "}"
+# definitions   ::= { TypeId "=" type }
+# type          ::= product | sum
+# product       ::= fields ["attributes" fields]
+# fields        ::= "(" { field, "," } field ")"
+# field         ::= TypeId ["?" | "*"] [Id]
+# sum           ::= constructor { "|" constructor } ["attributes" fields]
+# constructor   ::= ConstructorId [fields]
+#
+# [1] "The Zephyr Abstract Syntax Description Language" by Wang, et. al. See
+#     http://asdl.sourceforge.net/
+#-------------------------------------------------------------------------------
+from collections import namedtuple
+import re
 
-See http://asdl.sourceforge.net/ and
-http://www.cs.princeton.edu/~danwang/Papers/dsl97/dsl97-abstract.html.
+__all__ = [
+    'builtin_types', 'parse', 'AST', 'Module', 'Type', 'Constructor',
+    'Field', 'Sum', 'Product', 'VisitorBase', 'Check', 'check']
 
-Only supports top level module decl, not view.  I'm guessing that view
-is intended to support the browser and I'm not interested in the
-browser.
+# The following classes define nodes into which the ASDL description is parsed.
+# Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
+# structure used by a programming language. But ASDL files themselves need to be
+# parsed. This module parses ASDL files and uses a simple AST to represent them.
+# See the EBNF at the top of the file to understand the logical connection
+# between the various node types.
 
-Changes for Python: Add support for module versions
-"""
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
 
-import os
-import traceback
-
-import spark
-
-class Token(object):
-    # spark seems to dispatch in the parser based on a token's
-    # type attribute
-    def __init__(self, type, lineno):
-        self.type = type
-        self.lineno = lineno
-
-    def __str__(self):
-        return self.type
-
+class AST:
     def __repr__(self):
-        return str(self)
-
-class Id(Token):
-    def __init__(self, value, lineno):
-        self.type = 'Id'
-        self.value = value
-        self.lineno = lineno
-
-    def __str__(self):
-        return self.value
-
-class String(Token):
-    def __init__(self, value, lineno):
-        self.type = 'String'
-        self.value = value
-        self.lineno = lineno
-
-class ASDLSyntaxError(Exception):
-
-    def __init__(self, lineno, token=None, msg=None):
-        self.lineno = lineno
-        self.token = token
-        self.msg = msg
-
-    def __str__(self):
-        if self.msg is None:
-            return "Error at '%s', line %d" % (self.token, self.lineno)
-        else:
-            return "%s, line %d" % (self.msg, self.lineno)
-
-class ASDLScanner(spark.GenericScanner, object):
-
-    def tokenize(self, input):
-        self.rv = []
-        self.lineno = 1
-        super(ASDLScanner, self).tokenize(input)
-        return self.rv
-
-    def t_id(self, s):
-        r"[\w\.]+"
-        # XXX doesn't distinguish upper vs. lower, which is
-        # significant for ASDL.
-        self.rv.append(Id(s, self.lineno))
-
-    def t_string(self, s):
-        r'"[^"]*"'
-        self.rv.append(String(s, self.lineno))
-
-    def t_xxx(self, s): # not sure what this production means
-        r"<="
-        self.rv.append(Token(s, self.lineno))
-
-    def t_punctuation(self, s):
-        r"[\{\}\*\=\|\(\)\,\?\:]"
-        self.rv.append(Token(s, self.lineno))
-
-    def t_comment(self, s):
-        r"\-\-[^\n]*"
-        pass
-
-    def t_newline(self, s):
-        r"\n"
-        self.lineno += 1
-
-    def t_whitespace(self, s):
-        r"[ \t]+"
-        pass
-
-    def t_default(self, s):
-        r" . +"
-        raise ValueError, "unmatched input: %s" % `s`
-
-class ASDLParser(spark.GenericParser, object):
-    def __init__(self):
-        super(ASDLParser, self).__init__("module")
-
-    def typestring(self, tok):
-        return tok.type
-
-    def error(self, tok):
-        raise ASDLSyntaxError(tok.lineno, tok)
-
-    def p_module_0(self, (module, name, _0, _1)):
-        " module ::= Id Id { } "
-        if module.value != "module":
-            raise ASDLSyntaxError(module.lineno,
-                                  msg="expected 'module', found %s" % module)
-        return Module(name, None)
-
-    def p_module(self, (module, name, _0, definitions, _1)):
-        " module ::= Id Id { definitions } "
-        if module.value != "module":
-            raise ASDLSyntaxError(module.lineno,
-                                  msg="expected 'module', found %s" % module)
-        return Module(name, definitions)
-
-    def p_definition_0(self, (definition,)):
-        " definitions ::= definition "
-        return definition
-
-    def p_definition_1(self, (definitions, definition)):
-        " definitions ::= definition definitions "
-        return definitions + definition
-
-    def p_definition(self, (id, _, type)):
-        " definition ::= Id = type "
-        return [Type(id, type)]
-
-    def p_type_0(self, (product,)):
-        " type ::= product "
-        return product
-
-    def p_type_1(self, (sum,)):
-        " type ::= sum "
-        return Sum(sum)
-
-    def p_type_2(self, (sum, id, _0, attributes, _1)):
-        " type ::= sum Id ( fields ) "
-        if id.value != "attributes":
-            raise ASDLSyntaxError(id.lineno,
-                                  msg="expected attributes, found %s" % id)
-        if attributes:
-            attributes.reverse()
-        return Sum(sum, attributes)
-
-    def p_product(self, (_0, fields, _1)):
-        " product ::= ( fields ) "
-        # XXX can't I just construct things in the right order?
-        fields.reverse()
-        return Product(fields)
-
-    def p_sum_0(self, (constructor,)):
-        " sum ::= constructor "
-        return [constructor]
-
-    def p_sum_1(self, (constructor, _, sum)):
-        " sum ::= constructor | sum "
-        return [constructor] + sum
-
-    def p_sum_2(self, (constructor, _, sum)):
-        " sum ::= constructor | sum "
-        return [constructor] + sum
-
-    def p_constructor_0(self, (id,)):
-        " constructor ::= Id "
-        return Constructor(id)
-
-    def p_constructor_1(self, (id, _0, fields, _1)):
-        " constructor ::= Id ( fields ) "
-        # XXX can't I just construct things in the right order?
-        fields.reverse()
-        return Constructor(id, fields)
-
-    def p_fields_0(self, (field,)):
-        " fields ::= field "
-        return [field]
-
-    def p_fields_1(self, (field, _, fields)):
-        " fields ::= field , fields "
-        return fields + [field]
-
-    def p_field_0(self, (type,)):
-        " field ::= Id "
-        return Field(type)
-
-    def p_field_1(self, (type, name)):
-        " field ::= Id Id "
-        return Field(type, name)
-
-    def p_field_2(self, (type, _, name)):
-        " field ::= Id * Id "
-        return Field(type, name, seq=True)
-
-    def p_field_3(self, (type, _, name)):
-        " field ::= Id ? Id "
-        return Field(type, name, opt=True)
-
-    def p_field_4(self, (type, _)):
-        " field ::= Id * "
-        return Field(type, seq=True)
-
-    def p_field_5(self, (type, _)):
-        " field ::= Id ? "
-        return Field(type, opt=True)
-
-builtin_types = ("identifier", "string", "int", "bool", "object")
-
-# below is a collection of classes to capture the AST of an AST :-)
-# not sure if any of the methods are useful yet, but I'm adding them
-# piecemeal as they seem helpful
-
-class AST(object):
-    pass # a marker class
+        raise NotImplementedError
 
 class Module(AST):
     def __init__(self, name, dfns):
         self.name = name
         self.dfns = dfns
-        self.types = {} # maps type name to value (from dfns)
-        for type in dfns:
-            self.types[type.name.value] = type.value
+        self.types = {type.name: type.value for type in dfns}
 
     def __repr__(self):
-        return "Module(%s, %s)" % (self.name, self.dfns)
+        return 'Module({0.name}, {0.dfns})'.format(self)
 
 class Type(AST):
     def __init__(self, name, value):
@@ -237,7 +54,7 @@
         self.value = value
 
     def __repr__(self):
-        return "Type(%s, %s)" % (self.name, self.value)
+        return 'Type({0.name}, {0.value})'.format(self)
 
 class Constructor(AST):
     def __init__(self, name, fields=None):
@@ -245,7 +62,7 @@
         self.fields = fields or []
 
     def __repr__(self):
-        return "Constructor(%s, %s)" % (self.name, self.fields)
+        return 'Constructor({0.name}, {0.fields})'.format(self)
 
 class Field(AST):
     def __init__(self, type, name=None, seq=False, opt=False):
@@ -262,9 +79,9 @@
         else:
             extra = ""
         if self.name is None:
-            return "Field(%s%s)" % (self.type, extra)
+            return 'Field({0.type}{1})'.format(self, extra)
         else:
-            return "Field(%s, %s%s)" % (self.type, self.name, extra)
+            return 'Field({0.type}, {0.name}{1})'.format(self, extra)
 
 class Sum(AST):
     def __init__(self, types, attributes=None):
@@ -272,47 +89,54 @@
         self.attributes = attributes or []
 
     def __repr__(self):
-        if self.attributes is None:
-            return "Sum(%s)" % self.types
+        if self.attributes:
+            return 'Sum({0.types}, {0.attributes})'.format(self)
         else:
-            return "Sum(%s, %s)" % (self.types, self.attributes)
+            return 'Sum({0.types})'.format(self)
 
 class Product(AST):
-    def __init__(self, fields):
+    def __init__(self, fields, attributes=None):
         self.fields = fields
+        self.attributes = attributes or []
 
     def __repr__(self):
-        return "Product(%s)" % self.fields
+        if self.attributes:
+            return 'Product({0.fields}, {0.attributes})'.format(self)
+        else:
+            return 'Product({0.fields})'.format(self)
+
+# A generic visitor for the meta-AST that describes ASDL. This can be used by
+# emitters. Note that this visitor does not provide a generic visit method, so a
+# subclass needs to define visit methods from visitModule to as deep as the
+# interesting node.
+# We also define a Check visitor that makes sure the parsed ASDL is well-formed.
 
 class VisitorBase(object):
+    """Generic tree visitor for ASTs."""
+    def __init__(self):
+        self.cache = {}
 
-    def __init__(self, skip=False):
-        self.cache = {}
-        self.skip = skip
-
-    def visit(self, object, *args):
-        meth = self._dispatch(object)
-        if meth is None:
-            return
-        meth(object, *args)
-
-    def _dispatch(self, object):
-        assert isinstance(object, AST), repr(object)
-        klass = object.__class__
+    def visit(self, obj, *args):
+        klass = obj.__class__
         meth = self.cache.get(klass)
         if meth is None:
             methname = "visit" + klass.__name__
-            if self.skip:
-                meth = getattr(self, methname, None)
-            else:
-                meth = getattr(self, methname)
+            meth = getattr(self, methname, None)
             self.cache[klass] = meth
-        return meth
+        if meth:
+            try:
+                meth(obj, *args)
+            except Exception as e:
+                print("Error visiting %r: %s" % (obj, e))
+                raise
 
 class Check(VisitorBase):
+    """A visitor that checks a parsed ASDL tree for correctness.
 
+    Errors are printed and accumulated.
+    """
     def __init__(self):
-        super(Check, self).__init__(skip=True)
+        super(Check, self).__init__()
         self.cons = {}
         self.errors = 0
         self.types = {}
@@ -334,8 +158,8 @@
         if conflict is None:
             self.cons[key] = name
         else:
-            print "Redefinition of constructor %s" % key
-            print "Defined in %s and %s" % (conflict, name)
+            print('Redefinition of constructor {}'.format(key))
+            print('Defined in {} and {}'.format(conflict, name))
             self.errors += 1
         for f in cons.fields:
             self.visit(f, key)
@@ -350,6 +174,11 @@
             self.visit(f, name)
 
 def check(mod):
+    """Check the parsed ASDL tree for correctness.
+
+    Return True if success. For failure, the errors are printed out and False
+    is returned.
+    """
     v = Check()
     v.visit(mod)
 
@@ -357,40 +186,190 @@
         if t not in mod.types and not t in builtin_types:
             v.errors += 1
             uses = ", ".join(v.types[t])
-            print "Undefined type %s, used in %s" % (t, uses)
-
+            print('Undefined type {}, used in {}'.format(t, uses))
     return not v.errors
 
-def parse(file):
-    scanner = ASDLScanner()
-    parser = ASDLParser()
+# The ASDL parser itself comes next. The only interesting external interface
+# here is the top-level parse function.
 
-    buf = open(file).read()
-    tokens = scanner.tokenize(buf)
-    try:
-        return parser.parse(tokens)
-    except ASDLSyntaxError, err:
-        print err
-        lines = buf.split("\n")
-        print lines[err.lineno - 1] # lines starts at 0, files at 1
+def parse(filename):
+    """Parse ASDL from the given file and return a Module node describing it."""
+    with open(filename) as f:
+        parser = ASDLParser()
+        return parser.parse(f.read())
 
-if __name__ == "__main__":
-    import glob
-    import sys
+# Types for describing tokens in an ASDL specification.
+class TokenKind:
+    """TokenKind is provides a scope for enumerated token kinds."""
+    (ConstructorId, TypeId, Equals, Comma, Question, Pipe, Asterisk,
+     LParen, RParen, LBrace, RBrace) = range(11)
 
-    if len(sys.argv) > 1:
-        files = sys.argv[1:]
-    else:
-        testdir = "tests"
-        files = glob.glob(testdir + "/*.asdl")
+    operator_table = {
+        '=': Equals, ',': Comma,    '?': Question, '|': Pipe,    '(': LParen,
+        ')': RParen, '*': Asterisk, '{': LBrace,   '}': RBrace}
 
-    for file in files:
-        print file
-        mod = parse(file)
-        print "module", mod.name
-        print len(mod.dfns), "definitions"
-        if not check(mod):
-            print "Check failed"
+Token = namedtuple('Token', 'kind value lineno')
+
+class ASDLSyntaxError(Exception):
+    def __init__(self, msg, lineno=None):
+        self.msg = msg
+        self.lineno = lineno or ''
+
+    def __str__(self):
+        return 'Syntax error on line {0.lineno}: {0.msg}'.format(self)
+
+def tokenize_asdl(buf):
+    """Tokenize the given buffer. Yield Token objects."""
+    for lineno, line in enumerate(buf.splitlines(), 1):
+        for m in re.finditer(r'\s*(\w+|--.*|.)', line.strip()):
+            c = m.group(1)
+            if c[0].isalpha():
+                # Some kind of identifier
+                if c[0].isupper():
+                    yield Token(TokenKind.ConstructorId, c, lineno)
+                else:
+                    yield Token(TokenKind.TypeId, c, lineno)
+            elif c[:2] == '--':
+                # Comment
+                break
+            else:
+                # Operators
+                try:
+                    op_kind = TokenKind.operator_table[c]
+                except KeyError:
+                    raise ASDLSyntaxError('Invalid operator %s' % c, lineno)
+                yield Token(op_kind, c, lineno)
+
+class ASDLParser:
+    """Parser for ASDL files.
+
+    Create, then call the parse method on a buffer containing ASDL.
+    This is a simple recursive descent parser that uses tokenize_asdl for the
+    lexing.
+    """
+    def __init__(self):
+        self._tokenizer = None
+        self.cur_token = None
+
+    def parse(self, buf):
+        """Parse the ASDL in the buffer and return an AST with a Module root.
+        """
+        self._tokenizer = tokenize_asdl(buf)
+        self._advance()
+        return self._parse_module()
+
+    def _parse_module(self):
+        if self._at_keyword('module'):
+            self._advance()
         else:
-            for dfn in mod.dfns:
-                print dfn.type
+            raise ASDLSyntaxError(
+                'Expected "module" (found {})'.format(self.cur_token.value),
+                self.cur_token.lineno)
+        name = self._match(self._id_kinds)
+        self._match(TokenKind.LBrace)
+        defs = self._parse_definitions()
+        self._match(TokenKind.RBrace)
+        return Module(name, defs)
+
+    def _parse_definitions(self):
+        defs = []
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            self._match(TokenKind.Equals)
+            type = self._parse_type()
+            defs.append(Type(typename, type))
+        return defs
+
+    def _parse_type(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            # If we see a (, it's a product
+            return self._parse_product()
+        else:
+            # Otherwise it's a sum. Look for ConstructorId
+            sumlist = [Constructor(self._match(TokenKind.ConstructorId),
+                                   self._parse_optional_fields())]
+            while self.cur_token.kind  == TokenKind.Pipe:
+                # More constructors
+                self._advance()
+                sumlist.append(Constructor(
+                                self._match(TokenKind.ConstructorId),
+                                self._parse_optional_fields()))
+            return Sum(sumlist, self._parse_optional_attributes())
+
+    def _parse_product(self):
+        return Product(self._parse_fields(), self._parse_optional_attributes())
+
+    def _parse_fields(self):
+        fields = []
+        self._match(TokenKind.LParen)
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            is_seq, is_opt = self._parse_optional_field_quantifier()
+            id = (self._advance() if self.cur_token.kind in self._id_kinds
+                                  else None)
+            fields.append(Field(typename, id, seq=is_seq, opt=is_opt))
+            if self.cur_token.kind == TokenKind.RParen:
+                break
+            elif self.cur_token.kind == TokenKind.Comma:
+                self._advance()
+        self._match(TokenKind.RParen)
+        return fields
+
+    def _parse_optional_fields(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_attributes(self):
+        if self._at_keyword('attributes'):
+            self._advance()
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_field_quantifier(self):
+        is_seq, is_opt = False, False
+        if self.cur_token.kind == TokenKind.Asterisk:
+            is_seq = True
+            self._advance()
+        elif self.cur_token.kind == TokenKind.Question:
+            is_opt = True
+            self._advance()
+        return is_seq, is_opt
+
+    def _advance(self):
+        """ Return the value of the current token and read the next one into
+            self.cur_token.
+        """
+        cur_val = None if self.cur_token is None else self.cur_token.value
+        try:
+            self.cur_token = next(self._tokenizer)
+        except StopIteration:
+            self.cur_token = None
+        return cur_val
+
+    _id_kinds = (TokenKind.ConstructorId, TokenKind.TypeId)
+
+    def _match(self, kind):
+        """The 'match' primitive of RD parsers.
+
+        * Verifies that the current token is of the given kind (kind can
+          be a tuple, in which the kind must match one of its members).
+        * Returns the value of the current token
+        * Reads in the next token
+        """
+        if (isinstance(kind, tuple) and self.cur_token.kind in kind or
+            self.cur_token.kind == kind
+            ):
+            value = self.cur_token.value
+            self._advance()
+            return value
+        else:
+            raise ASDLSyntaxError(
+                'Unmatched {} (found {})'.format(kind, self.cur_token.kind),
+                self.cur_token.lineno)
+
+    def _at_keyword(self, keyword):
+        return (self.cur_token.kind == TokenKind.TypeId and
+                self.cur_token.value == keyword)

From pypy.commits at gmail.com  Wed Jun  8 10:16:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 07:16:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Give an _attrs_ in the Resume class,
 so that if the corresponding
Message-ID: <575828c7.03c31c0a.1ef99.ffffc503@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85028:57689fe200ad
Date: 2016-06-08 16:17 +0200
http://bitbucket.org/pypy/pypy/changeset/57689fe200ad/

Log:	Give an _attrs_ in the Resume class, so that if the corresponding
	"yield" point is actually never reached, the Resume instance is
	never created, but the "getattr" operations don't make blocked
	blocks

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4610,6 +4610,19 @@
         a.build_types(fd, [])
         py.test.raises(AnnotatorError, a.build_types, fb, [])
 
+    def test_annotate_generator_with_unreachable_yields(self):
+        def f(n):
+            if n < 0:
+                yield 42
+            yield n
+            yield n
+        def main(n):
+            for x in f(abs(n)):
+                pass
+        #
+        a = self.RPythonAnnotator()
+        a.build_types(main, [int])
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -132,13 +132,14 @@
                 del block.operations[index]
                 newlink = split_block(block, index)
                 newblock = newlink.target
+                varnames = get_variable_names(newlink.args)
                 #
                 class Resume(AbstractPosition):
                     _immutable_ = True
+                    _attrs_ = varnames
                     block = newblock
                 Resume.__name__ = 'Resume%d' % len(mappings)
                 mappings.append(Resume)
-                varnames = get_variable_names(newlink.args)
                 #
                 _insert_reads(newblock, varnames)
                 #

From pypy.commits at gmail.com  Wed Jun  8 10:32:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 07:32:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Another test
Message-ID: <57582c99.832c1c0a.20d42.ffffd281@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85029:e5fe19dddeea
Date: 2016-06-08 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/e5fe19dddeea/

Log:	Another test

diff --git a/rpython/rtyper/test/test_generator.py b/rpython/rtyper/test/test_generator.py
--- a/rpython/rtyper/test/test_generator.py
+++ b/rpython/rtyper/test/test_generator.py
@@ -113,3 +113,21 @@
             return s
         res = self.interpret(g, [])
         assert res == 6
+
+    def test_generator_with_unreachable_yields(self):
+        def f(n):
+            if n < 0:
+                yield 42
+            yield n
+            if n < 0:
+                yield 43
+            yield n
+            if n < 0:
+                yield 44
+        def main(n):
+            y = 0
+            for x in f(abs(n)):
+                y += x
+            return y
+        res = self.interpret(main, [-100])
+        assert res == 200

From pypy.commits at gmail.com  Wed Jun  8 11:25:19 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 08 Jun 2016 08:25:19 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update for release,
 more consistant release naming
Message-ID: <575838df.426dc20a.8e816.ffffc328@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r755:78525e4abdef
Date: 2016-06-08 18:24 +0300
http://bitbucket.org/pypy/pypy.org/changeset/78525e4abdef/

Log:	update for release, more consistant release naming

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.

 
We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:

 
@@ -113,22 +113,22 @@
 degrees of being up-to-date.
 
 
-

-
Python2.7 compatible PyPy 5.1

+

+
Python2.7 compatible PyPy 5.3

 
@@ -197,7 +197,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in /opt, and
 if you want, put a symlink from somewhere like
-/usr/local/bin/pypy to /path/to/pypy-5.1.1/bin/pypy.  Do
+/usr/local/bin/pypy to /path/to/pypy2-5.3.0/bin/pypy.  Do
 not move or copy the executable pypy outside the tree – put
 a symlink to it, otherwise it will not find its libraries.

 

@@ -239,7 +239,7 @@
 
  • Get the source code.  The following packages contain the source at
 the same revision as the above binaries:
    
 
 
    Or you can checkout the current trunk using Mercurial (the trunk
 usually works and is of course more up-to-date):
    
@@ -283,9 +283,9 @@
 pypy ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone   # get the sandbox version
    • -

  • Enjoy Mandelbrot :-) It takes on the order of an hour to -finish the translation, and 2.x GB of RAM on a 32-bit system -and 4.x GB on 64-bit systems. (Do not start a translation on a +

  • Enjoy Mandelbrot :-) It takes on the order of half an hour to +finish the translation, and about 3GB of RAM on a 32-bit system +and about 5GB on 64-bit systems. (Do not start a translation on a machine with insufficient RAM! It will just swap forever. See notes below in that case.)

    • @@ -345,7 +345,9 @@

    Packaging

    -

    Once PyPy is translated from source the binary package similar to those provided in the section Default (with a JIT Compiler) above could be easily created with package.py script as following:

    +

    Once PyPy is translated from source the binary package similar to those +provided in the section Default (with a JIT Compiler) above could be +easily created with package.py script as following:

     cd ./pypy/pypy/tool/release/
 python package.py --help #for information
@@ -377,41 +379,6 @@
 
    
 
    Checksums
    
 
    Here are the checksums for each of the downloads
    
-
    pypy-5.1.2 (bugfix release for s390x) md5, sha1 and sha256:
    
-
    -00b3d541963c52c8f29fc5da60bfd24b  pypy-5.1.2-s390x.tar.bz2
-76b90b73e5433a965118af1217d824c1  pypy-5.1.2-src.tar.bz2
-55230ac6e819605cda5eb249c54a7d5aa20b435b  pypy-5.1.2-s390x.tar.bz2
-994b025d81f7a90c6bae31cf6a9fb2622fb52961  pypy-5.1.2-src.tar.bz2
-c1c7ef0c64addfc2d41f907235cd9d028f9fee25badcc08fc80d53cf7ffcd487  pypy-5.1.2-s390x.tar.bz2
-9b633f9f728701277bd69dfed7390fd826e9f3770e6599c03a4582c6acc57463  pypy-5.1.2-src.tar.bz2
-
    
-
    pypy-5.1.1 md5:
    
-
    -3fa98eb80ef5caa5a6f9d4468409a632  pypy-5.1.1-linux64.tar.bz2
-1d5874f076d18ecd4fd50054cca0c383  pypy-5.1.1-linux-armel.tar.bz2
-9e47e370d57293074bbef6c4c0c4736d  pypy-5.1.1-linux-armhf-raring.tar.bz2
-b6643215abc92ed8efd94e6205305a36  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-224e4d5870d88fb444d8f4f1791140e5  pypy-5.1.1-linux.tar.bz2
-e35510b39e34f1c2199c283bf8655e5c  pypy-5.1.1-osx64.tar.bz2
-9d8b82448416e0203efa325364f759e8  pypy-5.1.1-s390x.tar.bz2
-8c2630896178e650e593686ddae625ac  pypy-5.1.1-src.tar.bz2
-f70ee6096d567c549a2bf11484bfbd0b  pypy-5.1.1-src.zip
-d70b4385fbf0a5e5260f6b7bedb231d4  pypy-5.1.1-win32.zip
-
    
-
    pypy-5.1.0 md5:
    
-
    -17baf9db5200559b9d6c45ec8f60ea48  pypy-5.1.0-linux-armel.tar.bz2
-c0f360b601cd723031c0edc18b62f118  pypy-5.1.0-linux-armhf-raring.tar.bz2
-27e5e98ccbca5ebb5933147556a46f77  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-224d1f124393c96c98b9acbaf4f92078  pypy-5.1.0-linux.tar.bz2
-2a58aa928ae1cabc6a3309cf98f6182e  pypy-5.1.0-linux64.tar.bz2
-7f546940acb3ceebb5967697a9b05b65  pypy-5.1.0-osx64.tar.bz2
-f9362ffc2946efcaadcc40fdb2c43df7  pypy-5.1.0-s390x.tar.bz2
-d0a76859c83fb0427674273977086cb2  pypy-5.1.0-src.tar.bz2
-204273a21dbf71c0827966265c40eb7a  pypy-5.1.0-src.zip
-a1710ae6f15b567bf3c8fd608553ad48  pypy-5.1.0-win32.zip
-
    
 
    pypy2.7-v5.3 md5:
    
 
     05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
@@ -442,60 +409,6 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
    
-
    pypy-5.1.1 sha1:
    
-
    -9ffc1fe9dfeec77a705b0d1af257da7e87894f5a  pypy-5.1.1-linux64.tar.bz2
-e432b157bc4cd2b5a21810ff45fd9a1507e8b8bf  pypy-5.1.1-linux-armel.tar.bz2
-5ed85f83566a4de5838c8b549943cb79250386ad  pypy-5.1.1-linux-armhf-raring.tar.bz2
-ddd1c20e049fcbc01f2bd9173ad77033540722a9  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-6767056bb71081bce8fcee04de0d0be02d71d4f9  pypy-5.1.1-linux.tar.bz2
-734eb82489d57a3b2b55d6b83153b3972dc6781d  pypy-5.1.1-osx64.tar.bz2
-2440d613430f9dfc57bc8db5cfd087f1169ee2d0  pypy-5.1.1-s390x.tar.bz2
-830e0a2c43c518b8c2b33f4ae40ac72b25e6da02  pypy-5.1.1-src.tar.bz2
-bf4826218579f7339acfb70fa0e6107d3527b095  pypy-5.1.1-src.zip
-3694e37c1cf6a2a938c108ee69126e4f40a0886e  pypy-5.1.1-win32.zip
-
    
-
    pypy-5.1.0 sha1:
    
-
    -114d4f981956b83cfbc0a3c819fdac0b0550cd82  pypy-5.1.0-linux-armel.tar.bz2
-e3060f8fa765c317ec1ad6923f9ea595b9d411c3  pypy-5.1.0-linux-armhf-raring.tar.bz2
-8943448afd1fd3e89be0575f69c6f3be69f2efbc  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-229e7dbc130d2cc92be9d1cde88f2d6f7f28621b  pypy-5.1.0-linux.tar.bz2
-c959524ce180f801bdbcbee4ca038309e1c771dd  pypy-5.1.0-linux64.tar.bz2
-216a52e44f3642176cf05fc3b4c6e2cf8981e400  pypy-5.1.0-osx64.tar.bz2
-b696059359a780ad3c2641b14c989021d93015e8  pypy-5.1.0-s390x.tar.bz2
-c9c497836e6235af9fee2a98e4aeaa2bc3a29550  pypy-5.1.0-src.tar.bz2
-a184ef5ada93d53e8dc4a9850a9ed764bd661d7b  pypy-5.1.0-src.zip
-4daba0932afcc4755d93d55aa3cbdd851da9198d  pypy-5.1.0-win32.zip
-
    
-
    pypy-5.1.1 sha256:
    
-
    -c852622e8bc81618c137da35fcf57b2349b956c07b6fd853300846e3cefa64fc  pypy-5.1.1-linux64.tar.bz2
-062b33641c24dfc8c6b5af955c2ddf3815b471de0af4bfc343020651b94d13bf  pypy-5.1.1-linux-armel.tar.bz2
-c4bcdabccd15669ea44d1c715cd36b2ca55b340a27b63e1a92ef5ab6eb158a8d  pypy-5.1.1-linux-armhf-raring.tar.bz2
-fc2a1f8719a7eca5d85d0bdcf499c6ab7409fc32aa312435bcbe66950b47e863  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-7951fd2b87c9e621ec57c932c20da2b8a4a9e87d8daeb9e2b7373f9444219abc  pypy-5.1.1-linux.tar.bz2
-fe2bbb7cf95eb91b1724029f81e85d1dbb6025a2e9a005cfe7258fe07602f771  pypy-5.1.1-osx64.tar.bz2
-4acd1066e07eb668665b302bf8e9338b6df136082c5ce28c62b70c6bb1b5cf9f  pypy-5.1.1-s390x.tar.bz2
-ca3d943d7fbd78bb957ee9e5833ada4bb8506ac99a41b7628790e286a65ed2be  pypy-5.1.1-src.tar.bz2
-cdcc967da36cde5586839cc631ef0d9123e19d3ce71ccfba03c68ac887374884  pypy-5.1.1-src.zip
-22a780e328ef053e098f2edc2302957ac3119adf7bf11ff23e225931806e7bcd  pypy-5.1.1-win32.zip
-
    
-
    pypy-5.1.0 sha256:
    
-
    -ea7017449ff0630431866423220c3688fc55c1a0b80a96af0ae138dd0751b81c  pypy-5.1.0-linux-armel.tar.bz2
-a3e13083591bccc301fb974ff0a6c7e4ab4e611e4b31c0932898b981c794462b  pypy-5.1.0-linux-armhf-raring.tar.bz2
-3bfcd251b4f3fd1a09520b2741c647c364d16d50c82b813732a78ac60ccb2b69  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-2f6c521b5b3c1082eab58be78655aa01ec400d19baeec93c455864a7483b8744  pypy-5.1.0-linux.tar.bz2
-0e8913351d043a50740b98cb89d99852b8bd6d11225a41c8abfc0baf7084cbf6  pypy-5.1.0-linux64.tar.bz2
-7e270c66347158dd794c101c4817f742f760ed805aa0d10abe19ba4a78a75118  pypy-5.1.0-osx64.tar.bz2
-096827f2cb041f9decc5a2b0b8fc6b5fe0748f229b0419fd73982e0714a292cd  pypy-5.1.0-s390x.tar.bz2
-16bab9501e942c0704abbf9cd6c4e950c6a76dc226cf1e447ea084916aef4714  pypy-5.1.0-src.tar.bz2
-afc1c72651c90418b57692a5628481dd09a3d3172765fd206e8bcdac7b1bf02d  pypy-5.1.0-src.zip
-044e7f35223a443412b5948740e60e93069a6f8b0a72053cc9d472874bb1b6cc  pypy-5.1.0-win32.zip
-3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
-53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
-
    
 
    pypy2.7-5.3 sha1:
    
 
     401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
@@ -509,6 +422,18 @@
 18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
 076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
 
    
+
    pypy3.3-v5.2-alpha sha1:
    
+
    +03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
+ae62d0df060e245b30eb07f12c5bc2260695ac36  pypy3.3-v5.2.0-alpha1-linux64.tar.bz2
+cf09edf1232a7bbb475fb19c8e6080d590774c4e  pypy3.3-v5.2.0-alpha1-linux-armel.tar.bz2
+3bec09a599371d0aca5408022a9ff4600f801e78  pypy3.3-v5.2.0-alpha1-linux-armhf-raring.tar.bz2
+addfd4466e4dead5a4e620214a015a314bfee83e  pypy3.3-v5.2.0-alpha1-linux-armhf-raspbian.tar.bz2
+f4a3badfe4c70465e9a2a43fde19e7a92975bc20  pypy3.3-v5.2.0-alpha1-osx64.tar.bz2
+eb630112d27063ba336b1d11d083edcda98c3a1f  pypy3.3-v5.2.0-alpha1-s390x.tar.bz2
+4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
+d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
+
    
 
    pypy2.7-5.3 sha256:
    
 
     bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
@@ -522,18 +447,6 @@
 09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
 32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
 
    
-
    pypy3.3-v5.2-alpha sha1:
    
-
    -03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
-ae62d0df060e245b30eb07f12c5bc2260695ac36  pypy3.3-v5.2.0-alpha1-linux64.tar.bz2
-cf09edf1232a7bbb475fb19c8e6080d590774c4e  pypy3.3-v5.2.0-alpha1-linux-armel.tar.bz2
-3bec09a599371d0aca5408022a9ff4600f801e78  pypy3.3-v5.2.0-alpha1-linux-armhf-raring.tar.bz2
-addfd4466e4dead5a4e620214a015a314bfee83e  pypy3.3-v5.2.0-alpha1-linux-armhf-raspbian.tar.bz2
-f4a3badfe4c70465e9a2a43fde19e7a92975bc20  pypy3.3-v5.2.0-alpha1-osx64.tar.bz2
-eb630112d27063ba336b1d11d083edcda98c3a1f  pypy3.3-v5.2.0-alpha1-s390x.tar.bz2
-4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
-d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
-
    
 
    pypy3.3-v5.2-alpha sha256:
    
 
     351aec101bdedddae7ea1b63845a5654b1a95fc9393894ef84a66749f6945f17  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,13 +14,12 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release — **PyPy 5.1.1** — (`what's new in PyPy 5.1?`_ and `what's new in PyPy 5.1.1?`_ )
+* the Python2.7 compatible release — **PyPy2.7 v5.3.0** — (`what's new in PyPy2.7?`_ )
 * the Python3.3 compatible release — **PyPy3.3 v5.2-alpha** — (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy 5.1?: http://doc.pypy.org/en/latest/release-5.1.0.html
-.. _what's new in PyPy 5.1.1?: http://doc.pypy.org/en/latest/release-5.1.1.html
+.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.0.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html
 
 
@@ -74,7 +73,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 5.1
+Python2.7 compatible PyPy 5.3
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -93,18 +92,18 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-linux.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1++-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1+-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.2-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
@@ -204,7 +203,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy-5.1.1/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.3.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -260,9 +259,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy-5.1.1-src.tar.bz2`__ (sources)
+   * `pypy2-v5.3.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-5.1.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::
@@ -306,9 +305,9 @@
      pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version
      pypy ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone   # get the sandbox version
 
-5. Enjoy Mandelbrot ``:-)``  It takes on the order of an hour to
-   finish the translation, and 2.x GB of RAM on a 32-bit system
-   and 4.x GB on 64-bit systems.  (Do not start a translation on a
+5. Enjoy Mandelbrot ``:-)``  It takes on the order of half an hour to
+   finish the translation, and about 3GB of RAM on a 32-bit system
+   and about 5GB on 64-bit systems.  (Do not start a translation on a
    machine with insufficient RAM!  It will just swap forever.  See
    notes below in that case.)
 
@@ -378,7 +377,9 @@
 Packaging
 ---------
 
-Once PyPy is translated from source the binary package similar to those provided in the section `Default (with a JIT Compiler)`_ above could be easily created with ``package.py`` script as following::
+Once PyPy is translated from source the binary package similar to those 
+provided in the section `Default (with a JIT Compiler)`_ above could be 
+easily created with ``package.py`` script as following::
 
     cd ./pypy/pypy/tool/release/
     python package.py --help #for information
@@ -414,41 +415,6 @@
 
 Here are the checksums for each of the downloads
 
-pypy-5.1.2 (bugfix release for s390x) md5, sha1 and sha256::
-
-    00b3d541963c52c8f29fc5da60bfd24b  pypy-5.1.2-s390x.tar.bz2
-    76b90b73e5433a965118af1217d824c1  pypy-5.1.2-src.tar.bz2
-    55230ac6e819605cda5eb249c54a7d5aa20b435b  pypy-5.1.2-s390x.tar.bz2
-    994b025d81f7a90c6bae31cf6a9fb2622fb52961  pypy-5.1.2-src.tar.bz2
-    c1c7ef0c64addfc2d41f907235cd9d028f9fee25badcc08fc80d53cf7ffcd487  pypy-5.1.2-s390x.tar.bz2
-    9b633f9f728701277bd69dfed7390fd826e9f3770e6599c03a4582c6acc57463  pypy-5.1.2-src.tar.bz2
-
-pypy-5.1.1 md5::
-
-    3fa98eb80ef5caa5a6f9d4468409a632  pypy-5.1.1-linux64.tar.bz2
-    1d5874f076d18ecd4fd50054cca0c383  pypy-5.1.1-linux-armel.tar.bz2
-    9e47e370d57293074bbef6c4c0c4736d  pypy-5.1.1-linux-armhf-raring.tar.bz2
-    b6643215abc92ed8efd94e6205305a36  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-    224e4d5870d88fb444d8f4f1791140e5  pypy-5.1.1-linux.tar.bz2
-    e35510b39e34f1c2199c283bf8655e5c  pypy-5.1.1-osx64.tar.bz2
-    9d8b82448416e0203efa325364f759e8  pypy-5.1.1-s390x.tar.bz2
-    8c2630896178e650e593686ddae625ac  pypy-5.1.1-src.tar.bz2
-    f70ee6096d567c549a2bf11484bfbd0b  pypy-5.1.1-src.zip
-    d70b4385fbf0a5e5260f6b7bedb231d4  pypy-5.1.1-win32.zip
-
-pypy-5.1.0 md5::
-
-    17baf9db5200559b9d6c45ec8f60ea48  pypy-5.1.0-linux-armel.tar.bz2
-    c0f360b601cd723031c0edc18b62f118  pypy-5.1.0-linux-armhf-raring.tar.bz2
-    27e5e98ccbca5ebb5933147556a46f77  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-    224d1f124393c96c98b9acbaf4f92078  pypy-5.1.0-linux.tar.bz2
-    2a58aa928ae1cabc6a3309cf98f6182e  pypy-5.1.0-linux64.tar.bz2
-    7f546940acb3ceebb5967697a9b05b65  pypy-5.1.0-osx64.tar.bz2
-    f9362ffc2946efcaadcc40fdb2c43df7  pypy-5.1.0-s390x.tar.bz2
-    d0a76859c83fb0427674273977086cb2  pypy-5.1.0-src.tar.bz2
-    204273a21dbf71c0827966265c40eb7a  pypy-5.1.0-src.zip
-    a1710ae6f15b567bf3c8fd608553ad48  pypy-5.1.0-win32.zip
-
 pypy2.7-v5.3 md5::
 
     05078bcdd797a025223d5905e2a12332  pypy2-v5.3.0-linux32.tar.bz2
@@ -480,60 +446,6 @@
     009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
 
-pypy-5.1.1 sha1::
-
-    9ffc1fe9dfeec77a705b0d1af257da7e87894f5a  pypy-5.1.1-linux64.tar.bz2
-    e432b157bc4cd2b5a21810ff45fd9a1507e8b8bf  pypy-5.1.1-linux-armel.tar.bz2
-    5ed85f83566a4de5838c8b549943cb79250386ad  pypy-5.1.1-linux-armhf-raring.tar.bz2
-    ddd1c20e049fcbc01f2bd9173ad77033540722a9  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-    6767056bb71081bce8fcee04de0d0be02d71d4f9  pypy-5.1.1-linux.tar.bz2
-    734eb82489d57a3b2b55d6b83153b3972dc6781d  pypy-5.1.1-osx64.tar.bz2
-    2440d613430f9dfc57bc8db5cfd087f1169ee2d0  pypy-5.1.1-s390x.tar.bz2
-    830e0a2c43c518b8c2b33f4ae40ac72b25e6da02  pypy-5.1.1-src.tar.bz2
-    bf4826218579f7339acfb70fa0e6107d3527b095  pypy-5.1.1-src.zip
-    3694e37c1cf6a2a938c108ee69126e4f40a0886e  pypy-5.1.1-win32.zip
-
-pypy-5.1.0 sha1::
-
-    114d4f981956b83cfbc0a3c819fdac0b0550cd82  pypy-5.1.0-linux-armel.tar.bz2
-    e3060f8fa765c317ec1ad6923f9ea595b9d411c3  pypy-5.1.0-linux-armhf-raring.tar.bz2
-    8943448afd1fd3e89be0575f69c6f3be69f2efbc  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-    229e7dbc130d2cc92be9d1cde88f2d6f7f28621b  pypy-5.1.0-linux.tar.bz2
-    c959524ce180f801bdbcbee4ca038309e1c771dd  pypy-5.1.0-linux64.tar.bz2
-    216a52e44f3642176cf05fc3b4c6e2cf8981e400  pypy-5.1.0-osx64.tar.bz2
-    b696059359a780ad3c2641b14c989021d93015e8  pypy-5.1.0-s390x.tar.bz2
-    c9c497836e6235af9fee2a98e4aeaa2bc3a29550  pypy-5.1.0-src.tar.bz2
-    a184ef5ada93d53e8dc4a9850a9ed764bd661d7b  pypy-5.1.0-src.zip
-    4daba0932afcc4755d93d55aa3cbdd851da9198d  pypy-5.1.0-win32.zip
-
-pypy-5.1.1 sha256::
-
-    c852622e8bc81618c137da35fcf57b2349b956c07b6fd853300846e3cefa64fc  pypy-5.1.1-linux64.tar.bz2
-    062b33641c24dfc8c6b5af955c2ddf3815b471de0af4bfc343020651b94d13bf  pypy-5.1.1-linux-armel.tar.bz2
-    c4bcdabccd15669ea44d1c715cd36b2ca55b340a27b63e1a92ef5ab6eb158a8d  pypy-5.1.1-linux-armhf-raring.tar.bz2
-    fc2a1f8719a7eca5d85d0bdcf499c6ab7409fc32aa312435bcbe66950b47e863  pypy-5.1.1-linux-armhf-raspbian.tar.bz2
-    7951fd2b87c9e621ec57c932c20da2b8a4a9e87d8daeb9e2b7373f9444219abc  pypy-5.1.1-linux.tar.bz2
-    fe2bbb7cf95eb91b1724029f81e85d1dbb6025a2e9a005cfe7258fe07602f771  pypy-5.1.1-osx64.tar.bz2
-    4acd1066e07eb668665b302bf8e9338b6df136082c5ce28c62b70c6bb1b5cf9f  pypy-5.1.1-s390x.tar.bz2
-    ca3d943d7fbd78bb957ee9e5833ada4bb8506ac99a41b7628790e286a65ed2be  pypy-5.1.1-src.tar.bz2
-    cdcc967da36cde5586839cc631ef0d9123e19d3ce71ccfba03c68ac887374884  pypy-5.1.1-src.zip
-    22a780e328ef053e098f2edc2302957ac3119adf7bf11ff23e225931806e7bcd  pypy-5.1.1-win32.zip
-
-pypy-5.1.0 sha256::
-
-    ea7017449ff0630431866423220c3688fc55c1a0b80a96af0ae138dd0751b81c  pypy-5.1.0-linux-armel.tar.bz2
-    a3e13083591bccc301fb974ff0a6c7e4ab4e611e4b31c0932898b981c794462b  pypy-5.1.0-linux-armhf-raring.tar.bz2
-    3bfcd251b4f3fd1a09520b2741c647c364d16d50c82b813732a78ac60ccb2b69  pypy-5.1.0-linux-armhf-raspbian.tar.bz2
-    2f6c521b5b3c1082eab58be78655aa01ec400d19baeec93c455864a7483b8744  pypy-5.1.0-linux.tar.bz2
-    0e8913351d043a50740b98cb89d99852b8bd6d11225a41c8abfc0baf7084cbf6  pypy-5.1.0-linux64.tar.bz2
-    7e270c66347158dd794c101c4817f742f760ed805aa0d10abe19ba4a78a75118  pypy-5.1.0-osx64.tar.bz2
-    096827f2cb041f9decc5a2b0b8fc6b5fe0748f229b0419fd73982e0714a292cd  pypy-5.1.0-s390x.tar.bz2
-    16bab9501e942c0704abbf9cd6c4e950c6a76dc226cf1e447ea084916aef4714  pypy-5.1.0-src.tar.bz2
-    afc1c72651c90418b57692a5628481dd09a3d3172765fd206e8bcdac7b1bf02d  pypy-5.1.0-src.zip
-    044e7f35223a443412b5948740e60e93069a6f8b0a72053cc9d472874bb1b6cc  pypy-5.1.0-win32.zip
-    3373b1d51fc610b962e0b535087073f2cc921ab0269ba2896b140ab4a56588fd  pypy-5.0.1++-ppc64.tar.bz2
-    53d742504a78366b833c04bd83740336aa4ddfecffeff6b2fa8728fcd6b4c8af  pypy-5.0.1+-ppc64le.tar.bz2
-
 pypy2.7-5.3 sha1::
 
     401066f82c8a26dfb1e3421ae4b117655b55ee8d  pypy2-v5.3.0-linux32.tar.bz2
@@ -547,6 +459,18 @@
     18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
     076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
 
+pypy3.3-v5.2-alpha sha1::
+
+    03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
+    ae62d0df060e245b30eb07f12c5bc2260695ac36  pypy3.3-v5.2.0-alpha1-linux64.tar.bz2
+    cf09edf1232a7bbb475fb19c8e6080d590774c4e  pypy3.3-v5.2.0-alpha1-linux-armel.tar.bz2
+    3bec09a599371d0aca5408022a9ff4600f801e78  pypy3.3-v5.2.0-alpha1-linux-armhf-raring.tar.bz2
+    addfd4466e4dead5a4e620214a015a314bfee83e  pypy3.3-v5.2.0-alpha1-linux-armhf-raspbian.tar.bz2
+    f4a3badfe4c70465e9a2a43fde19e7a92975bc20  pypy3.3-v5.2.0-alpha1-osx64.tar.bz2
+    eb630112d27063ba336b1d11d083edcda98c3a1f  pypy3.3-v5.2.0-alpha1-s390x.tar.bz2
+    4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
+    d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
+
 pypy2.7-5.3 sha256::
 
     bd422fe9d0b7d525d1da3f32855b047bc39ba397d0cf708d8f4f96fe874424f2  pypy2-v5.3.0-linux32.tar.bz2
@@ -560,18 +484,6 @@
     09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
     32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
 
-pypy3.3-v5.2-alpha sha1::
-
-    03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
-    ae62d0df060e245b30eb07f12c5bc2260695ac36  pypy3.3-v5.2.0-alpha1-linux64.tar.bz2
-    cf09edf1232a7bbb475fb19c8e6080d590774c4e  pypy3.3-v5.2.0-alpha1-linux-armel.tar.bz2
-    3bec09a599371d0aca5408022a9ff4600f801e78  pypy3.3-v5.2.0-alpha1-linux-armhf-raring.tar.bz2
-    addfd4466e4dead5a4e620214a015a314bfee83e  pypy3.3-v5.2.0-alpha1-linux-armhf-raspbian.tar.bz2
-    f4a3badfe4c70465e9a2a43fde19e7a92975bc20  pypy3.3-v5.2.0-alpha1-osx64.tar.bz2
-    eb630112d27063ba336b1d11d083edcda98c3a1f  pypy3.3-v5.2.0-alpha1-s390x.tar.bz2
-    4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
-    d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
-
 pypy3.3-v5.2-alpha sha256::
 
     351aec101bdedddae7ea1b63845a5654b1a95fc9393894ef84a66749f6945f17  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2

From pypy.commits at gmail.com  Wed Jun  8 11:26:58 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 08 Jun 2016 08:26:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak script for release
Message-ID: <57583942.0654c20a.1573.ffffc645@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85030:3520cfc1504e
Date: 2016-06-08 18:26 +0300
http://bitbucket.org/pypy/pypy/changeset/3520cfc1504e/

Log:	tweak script for release

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -3,7 +3,7 @@
 min=3
 rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
@@ -34,17 +34,19 @@
 plat=win32
 wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip
 unzip pypy-c-jit-latest-$plat.zip
+rm pypy-c-jit-latest-$plat.zip
 mv pypy-c-jit-*-$plat $rel-$plat
-zip -r $rel-$plat.zip $rel-$plat
+zip -rq $rel-$plat.zip $rel-$plat
 rm -rf $rel-$plat
 
 # Do this after creating a tag, note the untarred directory is pypy-pypy-
 # so make sure there is not another one
 wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
 tar -xf $tagname.tar.bz2
+rm $tagname.tar.bz2
 mv pypy-pypy-* $rel-src
 tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src
-zip -r $rel-src.zip $rel-src
+zip -rq $rel-src.zip $rel-src
 rm -rf $rel-src
 
 # Print out the md5, sha1, sha256

From pypy.commits at gmail.com  Wed Jun  8 12:00:02 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 09:00:02 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Kill PyPyClassCollector
Message-ID: <57584102.d12f1c0a.9f64b.7d5b@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85031:01ebecff75ed
Date: 2016-06-08 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/01ebecff75ed/

Log:	Kill PyPyClassCollector

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -171,9 +171,5 @@
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    pass
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.function import Method
 from pypy.tool.pytest import appsupport
 from pypy.tool.pytest.objspace import gettestobjspace
-from pypy.conftest import PyPyClassCollector
 from inspect import getmro
 
 
@@ -122,7 +121,7 @@
             self.w_instance = space.call_function(w_class)
 
 
-class AppClassCollector(PyPyClassCollector):
+class AppClassCollector(py.test.Class):
     Instance = AppClassInstance
 
     def setup(self):

From pypy.commits at gmail.com  Wed Jun  8 12:00:03 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 09:00:03 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Apply applevel marker in the
 same way as interplevel
Message-ID: <57584103.4aa71c0a.9c37.0632@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85032:449d109357aa
Date: 2016-06-08 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/449d109357aa/

Log:	Apply applevel marker in the same way as interplevel

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -84,8 +84,11 @@
     if config.option.runappdirect:
         return
     for item in items:
-        if isinstance(item, py.test.Function) and not is_applevel(item):
-            item.add_marker('interplevel')
+        if isinstance(item, py.test.Function)
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
 
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -20,13 +20,8 @@
     def __init__(self, excinfo):
         self.excinfo = excinfo
 
-marker = py.test.mark.applevel
 
 class AppTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(AppTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
     def _prunetraceback(self, traceback):
         return traceback
 

From pypy.commits at gmail.com  Wed Jun  8 12:13:55 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 08 Jun 2016 09:13:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix typos (arigato)
Message-ID: <57584443.441ac20a.67e79.ffffa0e0@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85033:59e0c0b0b7f6
Date: 2016-06-08 19:13 +0300
http://bitbucket.org/pypy/pypy/changeset/59e0c0b0b7f6/

Log:	fix typos (arigato)

diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -176,10 +176,10 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
- * Share cpyext Py* function wrappers according to the signature, shrining the
-   translated libpypy.so by about 
+ * Share cpyext Py* function wrappers according to the signature, shrinking the
+   translated libpypy.so by about 10% (measured without the JIT)
 
-  * Compile c snippets with -Werror, and fix warnings it exposed
+ * Compile c snippets with -Werror, and fix warnings it exposed
 
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
 .. _Numpy: https://bitbucket.org/pypy/numpy

From pypy.commits at gmail.com  Wed Jun  8 12:57:47 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 08 Jun 2016 09:57:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: formatting
Message-ID: <57584e8b.07a6c20a.98cf5.ffffe7a9@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85034:2a515474e18d
Date: 2016-06-08 19:57 +0300
http://bitbucket.org/pypy/pypy/changeset/2a515474e18d/

Log:	formatting

diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -176,10 +176,10 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
- * Share cpyext Py* function wrappers according to the signature, shrinking the
-   translated libpypy.so by about 10% (measured without the JIT)
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
 
- * Compile c snippets with -Werror, and fix warnings it exposed
+  * Compile c snippets with -Werror, and fix warnings it exposed
 
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
 .. _Numpy: https://bitbucket.org/pypy/numpy

From pypy.commits at gmail.com  Wed Jun  8 13:01:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 10:01:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: Reintroduce some JIT support for
 int_floordiv() in a way that makes
Message-ID: <57584f84.4ccf1c0a.2e2ef.ffff82da@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85035:f91ba52ae29e
Date: 2016-06-08 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/f91ba52ae29e/

Log:	Reintroduce some JIT support for int_floordiv() in a way that makes
	it at least half efficient. Add rarithmetic.int_c_div() as an
	explicit interface.

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -521,6 +521,7 @@
     # XXX some of the following functions should not become residual calls
     # but be really compiled
     rewrite_op_int_abs                = _do_builtin_call
+    rewrite_op_int_floordiv           = _do_builtin_call
     rewrite_op_llong_abs              = _do_builtin_call
     rewrite_op_llong_floordiv         = _do_builtin_call
     rewrite_op_llong_mod              = _do_builtin_call
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -248,6 +248,19 @@
     mask = x >> (LONG_BIT - 1)
     return (x ^ mask) - mask
 
+
+def _ll_2_int_floordiv(x, y):
+    # this is used only if the RPython program uses llop.int_floordiv()
+    # explicitly.  For 'a // b', see _handle_int_special() in jtransform.py.
+    # This is the reverse of rpython.rtyper.rint.ll_int_floordiv(), i.e.
+    # the same logic as rpython.rtyper.lltypesystem.opimpl.op_int_floordiv
+    # but written in a no-branch style.
+    r = x // y
+    p = r * y
+    # the JIT knows that if x and y are both positive, this is just 'r'
+    return r + (((x ^ y) >> (LONG_BIT - 1)) & (p != x))
+
+
 def _ll_1_cast_uint_to_float(x):
     # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
     # (which is a residual call right now in the x86 backend)
@@ -417,6 +430,7 @@
 # in the following calls to builtins, the JIT is allowed to look inside:
 inline_calls_to = [
     ('int_abs',              [lltype.Signed],                lltype.Signed),
+    ('int_floordiv',         [lltype.Signed, lltype.Signed], lltype.Signed),
     ('ll_math.ll_math_sqrt', [lltype.Float],                 lltype.Float),
 ]
 
diff --git a/rpython/jit/codewriter/test/test_support.py b/rpython/jit/codewriter/test/test_support.py
--- a/rpython/jit/codewriter/test/test_support.py
+++ b/rpython/jit/codewriter/test/test_support.py
@@ -3,7 +3,6 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.flowspace.model import Variable, Constant, SpaceOperation
 from rpython.jit.codewriter.support import decode_builtin_call, LLtypeHelpers
-from rpython.jit.codewriter.support import _ll_1_int_abs
 
 def newconst(x):
     return Constant(x, lltype.typeOf(x))
@@ -136,6 +135,7 @@
     py.test.raises(AttributeError, func, llstr(None), p2)
 
 def test_int_abs():
+    from rpython.jit.codewriter.support import _ll_1_int_abs
     assert _ll_1_int_abs(0) == 0
     assert _ll_1_int_abs(1) == 1
     assert _ll_1_int_abs(10) == 10
@@ -143,3 +143,12 @@
     assert _ll_1_int_abs(-1) == 1
     assert _ll_1_int_abs(-10) == 10
     assert _ll_1_int_abs(-sys.maxint) == sys.maxint
+
+def test_int_floordiv():
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.jit.codewriter.support import _ll_2_int_floordiv
+    for x in range(-6, 7):
+        for y in range(-3, 4):
+            if y != 0:
+                assert (_ll_2_int_floordiv(x, y) ==
+                        llop.int_floordiv(lltype.Signed, x, y))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5252,6 +5252,47 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_xor_positive_is_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_lt(i1, 0)
+        guard_false(i3) []
+        i4 = int_xor(i0, i1)
+        i5 = int_lt(i4, 0)
+        guard_false(i5) []
+        jump(i4, i0)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_lt(i1, 0)
+        guard_false(i3) []
+        i4 = int_xor(i0, i1)
+        jump(i4, i0)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_positive_rshift_bits_minus_1(self):
+        ops = """
+        [i0]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_rshift(i2, %d)
+        escape_n(i3)
+        jump(i0)
+        """ % (LONG_BIT - 1,)
+        expected = """
+        [i0]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        escape_n(0)
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_same_arg(self):
         ops = """
         [i0]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -955,6 +955,23 @@
         res = self.meta_interp(f, [-5])
         assert res == 5+4+3+2+1+0+1+2+3+4+5+6+7+8+9
 
+    def test_int_c_div(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_div(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 // n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_DIV)
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -650,6 +650,18 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_force_ge_zero(lltype.Signed, n)
 
+def int_c_div(x, y):
+    """Return the result of the C-style 'x / y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).  The JIT implements it
+    with a Python-style division followed by correction code.  This
+    is not that bad, because the JIT removes the correction code if
+    x and y are both nonnegative, and if y is any nonnegative constant
+    then the division turns into a rshift or a mul.
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_floordiv(lltype.Signed, x, y)
+
 @objectmodel.specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rlib.rarithmetic import *
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
+from hypothesis import given, strategies
 import sys
 import py
 
@@ -393,6 +394,18 @@
     assert not int_between(1, 2, 2)
     assert not int_between(1, 1, 1)
 
+def test_int_force_ge_zero():
+    assert int_force_ge_zero(42) == 42
+    assert int_force_ge_zero(0) == 0
+    assert int_force_ge_zero(-42) == 0
+
+ at given(strategies.integers(min_value=0, max_value=sys.maxint),
+       strategies.integers(min_value=1, max_value=sys.maxint))
+def test_int_c_div(x, y):
+    assert int_c_div(~x, y) == -(abs(~x) // y)
+    assert int_c_div( x,-y) == -(x // y)
+    assert int_c_div(~x,-y) == +(abs(~x) // y)
+
 # these can't be prebuilt on 32bit
 U1 = r_ulonglong(0x0102030405060708L)
 U2 = r_ulonglong(0x0807060504030201L)

From pypy.commits at gmail.com  Wed Jun  8 13:24:11 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 10:24:11 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: fix typo
Message-ID: <575854bb.c39dc20a.23aa1.fffff9b0@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85036:867128db4226
Date: 2016-06-08 18:23 +0100
http://bitbucket.org/pypy/pypy/changeset/867128db4226/

Log:	fix typo

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -84,7 +84,7 @@
     if config.option.runappdirect:
         return
     for item in items:
-        if isinstance(item, py.test.Function)
+        if isinstance(item, py.test.Function):
             if is_applevel(item):
                 item.add_marker('applevel')
             else:

From pypy.commits at gmail.com  Wed Jun  8 13:57:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 10:57:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Reintroduce some JIT support for
 int_mod(). Add rarithmetic.int_c_mod().
Message-ID: <57585c6c.03a81c0a.c6cc5.2f91@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85037:f0e47a0437e8
Date: 2016-06-08 19:47 +0200
http://bitbucket.org/pypy/pypy/changeset/f0e47a0437e8/

Log:	Reintroduce some JIT support for int_mod(). Add
	rarithmetic.int_c_mod().

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -522,6 +522,7 @@
     # but be really compiled
     rewrite_op_int_abs                = _do_builtin_call
     rewrite_op_int_floordiv           = _do_builtin_call
+    rewrite_op_int_mod                = _do_builtin_call
     rewrite_op_llong_abs              = _do_builtin_call
     rewrite_op_llong_floordiv         = _do_builtin_call
     rewrite_op_llong_mod              = _do_builtin_call
@@ -531,7 +532,6 @@
     rewrite_op_gc_id                  = _do_builtin_call
     rewrite_op_gc_pin                 = _do_builtin_call
     rewrite_op_gc_unpin               = _do_builtin_call
-    rewrite_op_uint_mod               = _do_builtin_call
     rewrite_op_cast_float_to_uint     = _do_builtin_call
     rewrite_op_cast_uint_to_float     = _do_builtin_call
     rewrite_op_weakref_create         = _do_builtin_call
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -260,6 +260,13 @@
     # the JIT knows that if x and y are both positive, this is just 'r'
     return r + (((x ^ y) >> (LONG_BIT - 1)) & (p != x))
 
+def _ll_2_int_mod(x, y):
+    # same comments as _ll_2_int_floordiv()
+    r = x % y
+    # the JIT knows that if x and y are both positive, this doesn't change 'r'
+    r -= y & (((x ^ y) & (r | -r)) >> (LONG_BIT - 1))
+    return r
+
 
 def _ll_1_cast_uint_to_float(x):
     # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
@@ -431,6 +438,7 @@
 inline_calls_to = [
     ('int_abs',              [lltype.Signed],                lltype.Signed),
     ('int_floordiv',         [lltype.Signed, lltype.Signed], lltype.Signed),
+    ('int_mod',              [lltype.Signed, lltype.Signed], lltype.Signed),
     ('ll_math.ll_math_sqrt', [lltype.Float],                 lltype.Float),
 ]
 
diff --git a/rpython/jit/codewriter/test/test_support.py b/rpython/jit/codewriter/test/test_support.py
--- a/rpython/jit/codewriter/test/test_support.py
+++ b/rpython/jit/codewriter/test/test_support.py
@@ -144,11 +144,13 @@
     assert _ll_1_int_abs(-10) == 10
     assert _ll_1_int_abs(-sys.maxint) == sys.maxint
 
-def test_int_floordiv():
+def test_int_floordiv_mod():
     from rpython.rtyper.lltypesystem.lloperation import llop
-    from rpython.jit.codewriter.support import _ll_2_int_floordiv
+    from rpython.jit.codewriter.support import _ll_2_int_floordiv, _ll_2_int_mod
     for x in range(-6, 7):
         for y in range(-3, 4):
             if y != 0:
                 assert (_ll_2_int_floordiv(x, y) ==
                         llop.int_floordiv(lltype.Signed, x, y))
+                assert (_ll_2_int_mod(x, y) ==
+                        llop.int_mod(lltype.Signed, x, y))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -97,17 +97,14 @@
         self.emit_operation(op)
 
         r = self.getintbound(op)
-        if b2.is_constant():
-            val = b2.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.is_constant():
-            val = b1.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)):
-            lesser = min(b1.upper, b2.upper)
-            r.intersect(IntBound(0, next_pow2_m1(lesser)))
+        pos1 = b1.known_ge(IntBound(0, 0))
+        pos2 = b2.known_ge(IntBound(0, 0))
+        if pos1 or pos2:
+            r.make_ge(IntBound(0, 0))
+        if pos1:
+            r.make_le(b1)
+        if pos2:
+            r.make_le(b2)
 
     def optimize_INT_SUB(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5188,6 +5188,25 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_and_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        i4 = int_ge(i3, 0)
+        guard_true(i4) []
+        jump(i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        jump(i3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_cmp_above_bounds(self):
         ops = """
         [p0,p1]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -972,6 +972,41 @@
         assert res == expected
         # should contain a call_i(..., OS=OS_INT_PY_DIV)
 
+    def test_int_c_mod(self):
+        from rpython.rlib.rarithmetic import int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_mod(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_MOD)
+
+    def test_positive_c_div_mod(self):
+        from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                assert i > 0
+                t += int_c_div(100, i) - int_c_mod(100, i)
+                i += 1
+            return t
+        expected = sum([100 // n - 100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # all the correction code should be dead now, xxx test that
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -662,6 +662,14 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_floordiv(lltype.Signed, x, y)
 
+def int_c_mod(x, y):
+    """Return the result of the C-style 'x % y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_mod(lltype.Signed, x, y)
+
 @objectmodel.specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -401,10 +401,13 @@
 
 @given(strategies.integers(min_value=0, max_value=sys.maxint),
        strategies.integers(min_value=1, max_value=sys.maxint))
-def test_int_c_div(x, y):
+def test_int_c_div_mod(x, y):
     assert int_c_div(~x, y) == -(abs(~x) // y)
     assert int_c_div( x,-y) == -(x // y)
     assert int_c_div(~x,-y) == +(abs(~x) // y)
+    for x1 in [x, ~x]:
+        for y1 in [y, -y]:
+            assert int_c_div(x1, y1) * y1 + int_c_mod(x1, y1) == x1
 
 # these can't be prebuilt on 32bit
 U1 = r_ulonglong(0x0102030405060708L)

From pypy.commits at gmail.com  Wed Jun  8 13:57:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 10:57:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Use rarithmetic.int_c_{div,mod}
Message-ID: <57585c7e.e976c20a.17417.fffffec8@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85038:b3c90ecc0170
Date: 2016-06-08 19:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b3c90ecc0170/

Log:	Use rarithmetic.int_c_{div,mod}

diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):

From pypy.commits at gmail.com  Wed Jun  8 14:15:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 11:15:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Another passing test
Message-ID: <575860bf.541a1c0a.c9402.3d18@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85039:940435e5feac
Date: 2016-06-08 20:15 +0200
http://bitbucket.org/pypy/pypy/changeset/940435e5feac/

Log:	Another passing test

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -1007,6 +1007,23 @@
         assert res == expected
         # all the correction code should be dead now, xxx test that
 
+    def test_int_c_div_by_constant(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = ['k'], reds = ['i', 't'])
+        def f(i, k):
+            t = 0
+            while i < 100:
+                myjitdriver.can_enter_jit(i=i, t=t, k=k)
+                myjitdriver.jit_merge_point(i=i, t=t, k=k)
+                t += int_c_div(i, k)
+                i += 1
+            return t
+        expected = sum([i // 10 for i in range(51, 100)])
+        assert f(-50, 10) == expected
+        res = self.meta_interp(f, [-50, 10])
+        assert res == expected
+        self.check_resops(call=0, uint_mul_high=2)
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):

From pypy.commits at gmail.com  Wed Jun  8 14:26:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 11:26:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57586359.442cc20a.d70fb.0a50@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85040:ba87481c5e04
Date: 2016-06-08 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/ba87481c5e04/

Log:	hg merge default

diff too long, truncating to 2000 out of 3831 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,17 +104,20 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas Stührk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -140,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +175,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,11 +210,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -228,7 +230,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -270,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +297,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -465,6 +463,7 @@
             else:
                 def __nonzero__(self):
                     return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,4 +1,5 @@
-# Empty
+import os
+pypydir = os.path.realpath(os.path.dirname(__file__))
 
 # XXX Should be empty again, soon.
 # XXX hack for win64:
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -18,8 +18,6 @@
 #
 option = None
 
-pypydir = os.path.realpath(os.path.dirname(__file__))
-
 def braindead_deindent(self):
     """monkeypatch that wont end up doing stupid in the python tokenizer"""
     text = '\n'.join(self.lines)
@@ -184,9 +182,6 @@
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
 
 class PyPyClassCollector(py.test.collect.Class):
     # All pypy Test classes have a "space" member.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -13,17 +13,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,9 +63,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -74,31 +74,34 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Edd Barrett
   Andreas Stührk
-  Edd Barrett
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -110,7 +113,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -126,11 +128,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -141,9 +145,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -153,8 +157,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -178,11 +180,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -198,7 +200,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -240,8 +241,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -265,9 +267,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
    release-5.0.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
    whatsnew-4.0.1.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,15 +53,17 @@
 immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated.
 
 
-Numpy improvements
-------------------
+NumPy rebooted
+--------------
 
-The numpy is rapidly progressing in pypy, so feel free to come to IRC and
-ask for proposed topic. A not necesarilly up-to-date `list of topics`_
-is also available.
+Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
+Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+test suite. We could use help analyzing the failures and fixing them either
+as patches to upstream NumPy, or as fixes to PyPy.
 
-.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt
-
+We also are looking for help in how to hijack NumPy dtype conversion and
+ufunc calls to allow the JIT to make them fast, using our internal _numpypy
+module.
 
 Improving the jitviewer
 ------------------------
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -2,15 +2,18 @@
 PyPy2.7 v5.3
 ============
 
-We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. 
-This release includes further improvements for the CAPI compatibility layer
-which we call cpyext. In addtion to complete support for lxml, we now pass
-most (more than 90%) of the upstream numpy test suite, and much of SciPy is
-supported as well.
+We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after
+`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3
+compatibility. This new PyPy2.7 release includes further improvements for the
+CAPI compatibility layer which we call cpyext. In addtion to complete support
+for lxml, we now pass most (more than 90%) of the upstream numpy test suite,
+and much of SciPy is supported as well.
 
-We also improved the speed of ... and ...
+We updated cffi_ to version 1.7 (small changes, documented here_).
 
-We updated cffi_ to ...
+.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html
+.. _cffi: https://cffi.readthedocs.org
+.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html
 
 You can download the PyPy2.7 v5.3 release here:
 
@@ -29,10 +32,6 @@
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
-.. _`numpy`: https://bitbucket.org/pypy/numpy
-.. _cffi: https://cffi.readthedocs.org
-.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html
-.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html
 
 What is PyPy?
 =============
@@ -41,13 +40,13 @@
 CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
 due to its integrated tracing JIT compiler.
 
-We also welcome developers of other
-`dynamic languages`_ to see what RPython can do for them.
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
 
 This release supports: 
 
   * **x86** machines on most common operating systems
-    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
   
   * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
   
@@ -65,6 +64,7 @@
 
   * Merge a major expansion of the C-API support in cpyext, here are some of
     the highlights:
+
       - allow c-snippet tests to be run with -A so we can verify we are compatible
       - fix many edge cases exposed by fixing tests to run with -A
       - issequence() logic matches cpython
@@ -72,7 +72,7 @@
       - add prelminary support for PyDateTime_*
       - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
         PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-      - PyAnySet_CheckExact, PyUnicode_Concat
+        PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo
       - improve support for PyGILState_Ensure, PyGILState_Release, and thread
         primitives, also find a case where CPython will allow thread creation
         before PyEval_InitThreads is run, dissallow on PyPy 
@@ -80,6 +80,10 @@
       - rewrite slot assignment for typeobjects
       - improve tracking of PyObject to rpython object mapping
       - support tp_as_{number, sequence, mapping, buffer} slots
+      - support ByteArrayObject via the new resizable_list_supporting_raw_ptr
+      - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's
+      - fix the signature of PyUFunc_FromFuncAndDataAndSignature
+      - implement many PyWhatever_FOO() as a macro taking a `void *`
 
   * CPyExt tweak: instead of "GIL not held when a CPython C extension module
     calls PyXxx", we now silently acquire/release the GIL.  Helps with
@@ -93,8 +97,42 @@
   * Generalize cpyext old-style buffers to more than just str/buffer, add
     support for mmap
 
+  * Support command line -v to trace import statements
+
+  * Add rposix functions for PyPy3.3 support
+
+  * Give super an __init__ and a simple __new__ for CPython compatibility
+
+  * Revive traceviewer, a tool to use pygame to view traces
+
 * Bug Fixes
 
+  * Fix issue #2277: only special-case two exact lists in zip(), not list
+    subclasses, because an overridden __iter__() should be called (probably)
+
+  * Fix issue #2226: Another tweak in the incremental GC- this should ensure
+    that progress in the major GC occurs quickly enough in all cases.
+
+  * Clarify and refactor documentation on http://doc.pypy.org
+
+  * Use "must be unicode, not %T" in unicodedata TypeErrors.
+
+  * Manually reset sys.settrace() and sys.setprofile() when we're done running.
+    This is not exactly what CPython does, but if we get an exception, unlike
+    CPython, we call functions from the 'traceback' module, and these would
+    call more the trace/profile function.  That's unexpected and can lead
+    to more crashes at this point.
+
+  * Use the appropriate tp_dealloc on a subclass of a builtin type, and call
+    tp_new for a python-sublcass of a C-API type
+
+  * Fix for issue #2285 - rare vmprof segfaults on OS/X
+
+  * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc
+
+  * Fix issue #2311 - grab the `__future__` flags imported in the main script, in
+    `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console
+
   * Issues reported with our previous release were resolved_ after reports from users on
     our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
     #pypy
@@ -103,6 +141,9 @@
 
   * Implement ufunc.outer on numpypy
 
+  * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_
+    accordingly)
+
 * Performance improvements:
 
   * Use bitstrings to compress lists of descriptors that are attached to an
@@ -114,11 +155,20 @@
     can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
     negative.
 
+  * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())`
+
+  * Use the __builtin_add_overflow built-ins if they are available
+
+  * Rework the way registers are moved/spilled in before_call()
 
 * Internal refactorings:
 
+  * Refactor code to better support Python3-compatible syntax
+
+  * Document and refactor OperationError -> oefmt
+
   * Reduce the size of generated C sources during translation by 
-    refactoring function declarations
+    eliminating many many unused struct declarations (Issue #2281)
 
   * Remove a number of translation-time options that were not tested and
     never used. Also fix a performance bug in the method cache
@@ -126,10 +176,14 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
+
   * Compile c snippets with -Werror, and fix warnings it exposed
 
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
 .. _Numpy: https://bitbucket.org/pypy/numpy
+.. _`the repo`: https://bitbucket.org/pypy/numpy
 
 Please update, and continue to help us make PyPy better.
 
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -73,6 +73,8 @@
     'Richard Lancaster':['richardlancaster'],
     'William Leslie':['William ML Leslie'],
     'Spenser Bauman':['Spenser Andrew Bauman'],
+    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
+    'timo':['timo at eistee.fritz.box'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,145 +1,21 @@
 =========================
-What's new in PyPy 5.1+
+What's new in PyPy2.7 5.3+
 =========================
 
-.. this is a revision shortly after release-5.1
-.. startrev: aa60332382a1
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
 
-.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+.. branch: fix-gen-dfa
 
-.. branch: gcheader-decl
+Resolves an issue with the generator script to build the dfa for Python syntax.
 
-Reduce the size of generated C sources.
+.. branch: z196-support
 
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
 
-.. branch: remove-objspace-options
+.. branch: s390x-5.3-catchup
 
-Remove a number of options from the build process that were never tested and
-never set. Fix a performance bug in the method cache.
-
-.. branch: bitstring
-
-JIT: use bitstrings to compress the lists of read or written descrs
-that we attach to EffectInfo.  Fixes a problem we had in
-remove-objspace-options.
-
-.. branch: cpyext-for-merge
-
-Update cpyext C-API support After this branch, we are almost able to support 
-upstream numpy via cpyext, so we created (yet another) fork of numpy at 
-github.com/pypy/numpy with the needed changes. Among the significant changes 
-to cpyext:
-  - allow c-snippet tests to be run with -A so we can verify we are compatible
-  - fix many edge cases exposed by fixing tests to run with -A
-  - issequence() logic matches cpython
-  - make PyStringObject and PyUnicodeObject field names compatible with cpython
-  - add prelminary support for PyDateTime_*
-  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
-    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-  - PyAnySet_CheckExact, PyUnicode_Concat
-  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
-    primitives, also find a case where CPython will allow thread creation
-    before PyEval_InitThreads is run, dissallow on PyPy 
-  - create a PyObject-specific list strategy
-  - rewrite slot assignment for typeobjects
-  - improve tracking of PyObject to rpython object mapping
-  - support tp_as_{number, sequence, mapping, buffer} slots
-
-(makes the pypy-c bigger; this was fixed subsequently by the
-share-cpyext-cpython-api branch)
-
-.. branch: share-mapdict-methods-2
-
-Reduce generated code for subclasses by using the same function objects in all
-generated subclasses.
-
-.. branch: share-cpyext-cpython-api
-
-.. branch: cpyext-auto-gil
-
-CPyExt tweak: instead of "GIL not held when a CPython C extension module
-calls PyXxx", we now silently acquire/release the GIL.  Helps with
-CPython C extension modules that call some PyXxx() functions without
-holding the GIL (arguably, they are theorically buggy).
-
-.. branch: cpyext-test-A
-
-Get the cpyext tests to pass with "-A" (i.e. when tested directly with
-CPython).
-
-.. branch: oefmt
-
-.. branch: cpyext-werror
-
-Compile c snippets with -Werror in cpyext
-
-.. branch: gc-del-3
-
-Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
-It is a more flexible way to make RPython finalizers.
-
-.. branch: unpacking-cpython-shortcut
-
-.. branch: cleanups
-
-.. branch: cpyext-more-slots
-
-.. branch: use-gc-del-3
-
-Use the new rgc.FinalizerQueue mechanism to clean up the handling of
-``__del__`` methods.  Fixes notably issue #2287.  (All RPython
-subclasses of W_Root need to use FinalizerQueue now.)
-
-.. branch: ufunc-outer
-
-Implement ufunc.outer on numpypy
-
-.. branch: verbose-imports
-
-Support ``pypy -v``: verbose imports.  It does not log as much as
-cpython, but it should be enough to help when debugging package layout
-problems.
-
-.. branch: cpyext-macros-cast
-
-Fix some warnings when compiling CPython C extension modules
-
-.. branch: syntax_fix
-
-.. branch: remove-raisingops
-
-Remove most of the _ovf, _zer and _val operations from RPython.  Kills
-quite some code internally, and allows the JIT to do better
-optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
-can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
-negative.
-
-.. branch: cpyext-old-buffers
-
-Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
-
-.. branch: numpy-includes
-
-Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
-This allows building upstream numpy and scipy in pypy via cpyext
-
-.. branch: traceviewer-common-merge-point-formats
-
-Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
-
-.. branch: cpyext-pickle
-
-Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
-at cpyext import time
-
-.. branch: nonmovable-list
-
-Add a way to ask "give me a raw pointer to this list's
-items".  Only for resizable lists of primitives.  Turns the GcArray
-nonmovable, possibly making a copy of it first.
-
-.. branch: cpyext-ext
-
-Finish the work already partially merged in cpyext-for-merge. Adds support
-for ByteArrayObject using the nonmovable-list, which also enables
-buffer(bytearray()) 
+Implement the backend related changes for s390x.
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -0,0 +1,145 @@
+=========================
+What's new in PyPy2.7 5.3
+=========================
+
+.. this is a revision shortly after release-5.1
+.. startrev: aa60332382a1
+
+.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+
+.. branch: gcheader-decl
+
+Reduce the size of generated C sources.
+
+
+.. branch: remove-objspace-options
+
+Remove a number of options from the build process that were never tested and
+never set. Fix a performance bug in the method cache.
+
+.. branch: bitstring
+
+JIT: use bitstrings to compress the lists of read or written descrs
+that we attach to EffectInfo.  Fixes a problem we had in
+remove-objspace-options.
+
+.. branch: cpyext-for-merge
+
+Update cpyext C-API support After this branch, we are almost able to support 
+upstream numpy via cpyext, so we created (yet another) fork of numpy at 
+github.com/pypy/numpy with the needed changes. Among the significant changes 
+to cpyext:
+  - allow c-snippet tests to be run with -A so we can verify we are compatible
+  - fix many edge cases exposed by fixing tests to run with -A
+  - issequence() logic matches cpython
+  - make PyStringObject and PyUnicodeObject field names compatible with cpython
+  - add prelminary support for PyDateTime_*
+  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
+    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
+  - PyAnySet_CheckExact, PyUnicode_Concat
+  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
+    primitives, also find a case where CPython will allow thread creation
+    before PyEval_InitThreads is run, dissallow on PyPy 
+  - create a PyObject-specific list strategy
+  - rewrite slot assignment for typeobjects
+  - improve tracking of PyObject to rpython object mapping
+  - support tp_as_{number, sequence, mapping, buffer} slots
+
+(makes the pypy-c bigger; this was fixed subsequently by the
+share-cpyext-cpython-api branch)
+
+.. branch: share-mapdict-methods-2
+
+Reduce generated code for subclasses by using the same function objects in all
+generated subclasses.
+
+.. branch: share-cpyext-cpython-api
+
+.. branch: cpyext-auto-gil
+
+CPyExt tweak: instead of "GIL not held when a CPython C extension module
+calls PyXxx", we now silently acquire/release the GIL.  Helps with
+CPython C extension modules that call some PyXxx() functions without
+holding the GIL (arguably, they are theorically buggy).
+
+.. branch: cpyext-test-A
+
+Get the cpyext tests to pass with "-A" (i.e. when tested directly with
+CPython).
+
+.. branch: oefmt
+
+.. branch: cpyext-werror
+
+Compile c snippets with -Werror in cpyext
+
+.. branch: gc-del-3
+
+Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
+It is a more flexible way to make RPython finalizers.
+
+.. branch: unpacking-cpython-shortcut
+
+.. branch: cleanups
+
+.. branch: cpyext-more-slots
+
+.. branch: use-gc-del-3
+
+Use the new rgc.FinalizerQueue mechanism to clean up the handling of
+``__del__`` methods.  Fixes notably issue #2287.  (All RPython
+subclasses of W_Root need to use FinalizerQueue now.)
+
+.. branch: ufunc-outer
+
+Implement ufunc.outer on numpypy
+
+.. branch: verbose-imports
+
+Support ``pypy -v``: verbose imports.  It does not log as much as
+cpython, but it should be enough to help when debugging package layout
+problems.
+
+.. branch: cpyext-macros-cast
+
+Fix some warnings when compiling CPython C extension modules
+
+.. branch: syntax_fix
+
+.. branch: remove-raisingops
+
+Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+quite some code internally, and allows the JIT to do better
+optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+negative.
+
+.. branch: cpyext-old-buffers
+
+Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
+
+.. branch: numpy-includes
+
+Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
+This allows building upstream numpy and scipy in pypy via cpyext
+
+.. branch: traceviewer-common-merge-point-formats
+
+Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
+
+.. branch: cpyext-pickle
+
+Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
+at cpyext import time
+
+.. branch: nonmovable-list
+
+Add a way to ask "give me a raw pointer to this list's
+items".  Only for resizable lists of primitives.  Turns the GcArray
+nonmovable, possibly making a copy of it first.
+
+.. branch: cpyext-ext
+
+Finish the work already partially merged in cpyext-for-merge. Adds support
+for ByteArrayObject using the nonmovable-list, which also enables
+buffer(bytearray()) 
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -10,7 +10,7 @@
 from rpython.config.config import ConflictConfigError
 from rpython.rlib import rlocale
 from pypy.tool.option import make_objspace
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.rlib import rthread
 from pypy.module.thread import os_thread
 
@@ -297,7 +297,7 @@
             self.hack_for_cffi_modules(driver)
 
         return self.get_entry_point(config)
-    
+
     def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
@@ -324,7 +324,7 @@
             while not basedir.join('include').exists():
                 _basedir = basedir.dirpath()
                 if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo', 
+                    raise ValueError('interpreter %s not inside pypy repo',
                                      str(exename))
                 basedir = _basedir
             modules = self.config.objspace.usemodules.getpaths()
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -6,7 +6,8 @@
 import sys, os, re, runpy, subprocess
 from rpython.tool.udir import udir
 from contextlib import contextmanager
-from pypy.conftest import PYTHON3, pypydir
+from pypy import pypydir
+from pypy.conftest import PYTHON3
 from pypy.interpreter.test.conftest import banner
 from lib_pypy._pypy_interact import irc_header
 
@@ -281,7 +282,7 @@
         child.expect('>>>')   # banner
         if irc_topic:
             assert irc_header in child.before
-        else:    
+        else:
             assert irc_header not in child.before
 
     def test_help(self):
@@ -1151,4 +1152,4 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-    
+
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -48,10 +48,10 @@
             return f.f_code
         assert g() is g.__code__
 
-    def test_f_trace_del(self): 
+    def test_f_trace_del(self):
         import sys
-        f = sys._getframe() 
-        del f.f_trace 
+        f = sys._getframe()
+        del f.f_trace
         assert f.f_trace is None
 
     def test_f_lineno(self):
@@ -120,7 +120,7 @@
         def f():
             assert sys._getframe().f_code.co_name == g()
         def g():
-            return sys._getframe().f_back.f_code.co_name 
+            return sys._getframe().f_back.f_code.co_name
         f()
 
     def test_f_back_virtualref(self):
@@ -237,7 +237,7 @@
     def test_trace_exc(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -300,7 +300,7 @@
     def test_trace_return_exc(self):
         import sys
         l = []
-        def trace(a,b,c): 
+        def trace(a,b,c):
             if b in ('exception', 'return'):
                 l.append((b, c))
             return trace
@@ -420,7 +420,7 @@
     def test_dont_trace_on_reraise(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -3,7 +3,7 @@
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
-class AppTestSlow:    
+class AppTestSlow:
     def setup_class(cls):
         if py.test.config.option.runappdirect:
             filename = __file__
@@ -62,7 +62,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('read_exc_type'),
                   space.wrap(read_exc_type_gw))
-    
+
 def _detach_helpers(space):
     space.delitem(space.builtin.w_dict,
                   space.wrap('hide_top_frame'))
@@ -90,7 +90,7 @@
         pckl = pickle.dumps(code)
         result = pickle.loads(pckl)
         assert code == result
-    
+
     def test_pickle_global_func(self):
         import types
         mod = types.ModuleType('mod')
@@ -107,7 +107,7 @@
             assert func is result
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_not_imported_module(self):
         import types
         mod = types.ModuleType('mod')
@@ -117,13 +117,13 @@
         result = pickle.loads(pckl)
         assert mod.__name__ == result.__name__
         assert mod.__dict__ == result.__dict__
-    
+
     def test_pickle_builtin_func(self):
         import pickle
         pckl = pickle.dumps(map)
         result = pickle.loads(pckl)
         assert map is result
-    
+
     def test_pickle_non_top_reachable_func(self):
         def func():
             return 42
@@ -140,7 +140,7 @@
         assert func.__dict__     == result.__dict__
         assert func.__doc__      == result.__doc__
         assert func.__globals__  == result.__globals__
-    
+
     def test_pickle_cell(self):
         def g():
             x = [42]
@@ -169,7 +169,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
         assert type(f1) is type(f2)
@@ -244,7 +244,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
     def test_frame_setstate_crash(self):
@@ -278,21 +278,21 @@
         pckl   = pickle.dumps(mod)
         result = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_moduledict(self):
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
         result   = pickle.loads(pckl)
         assert moddict is result
-    
+
     def test_pickle_bltins_module(self):
         import pickle
         mod  = __builtins__
         pckl     = pickle.dumps(mod)
         result   = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_buffer(self):
         skip("Can't pickle buffer objects on top of CPython either.  "
              "Do we really need it?")
@@ -301,14 +301,14 @@
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_complex(self):
         import pickle
         a = complex(1.23,4.567)
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_method(self):
         class myclass(object):
             def f(self):
@@ -329,7 +329,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_staticmethod(self):
         class myclass(object):
             def f():
@@ -340,7 +340,7 @@
         pckl     = pickle.dumps(method)
         result   = pickle.loads(pckl)
         assert method() == result()
-    
+
     def test_pickle_classmethod(self):
         class myclass(object):
             def f(cls):
@@ -358,7 +358,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_sequenceiter(self):
         '''
         In PyPy there is no distinction here between listiterator and
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.6.0"
+VERSION = "1.7.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -420,6 +420,14 @@
             w_result = ctype.ctitem.unpack_ptr(ctype, ptr, length)
         return w_result
 
+    def dir(self, space):
+        from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
+        ct = self.ctype
+        if isinstance(ct, W_CTypePointer):
+            ct = ct.ctitem
+        lst = ct.cdata_dir()
+        return space.newlist([space.wrap(s) for s in lst])
+
 
 class W_CDataMem(W_CData):
     """This is used only by the results of cffi.cast('int', x)
@@ -602,5 +610,6 @@
     __call__ = interp2app(W_CData.call),
     __iter__ = interp2app(W_CData.iter),
     __weakref__ = make_weakref_descr(W_CData),
+    __dir__ = interp2app(W_CData.dir),
     )
 W_CData.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -256,6 +256,9 @@
     def fget_elements(self, space): return self._fget('e')
     def fget_relements(self, space):return self._fget('R')
 
+    def cdata_dir(self):
+        return []
+
 
 W_CType.typedef = TypeDef(
     '_cffi_backend.CTypeDescr',
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -171,6 +171,12 @@
                 pass
         return W_CType.getcfield(self, attr)
 
+    def cdata_dir(self):
+        if self.size < 0:
+            return []
+        self.force_lazy_struct()
+        return self._fields_dict.keys()
+
 
 class W_CTypeStruct(W_CTypeStructOrUnion):
     kind = "struct"
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -201,6 +201,9 @@
         else:
             copy_string_to_raw(llstr(src_string), dest_data, 0, n)
     else:
+        # nowadays this case should be rare or impossible: as far as
+        # I know, all common types implementing the *writable* buffer
+        # interface now support get_raw_address()
         if src_is_ptr:
             for i in range(n):
                 dest_buf.setitem(i, src_data[i])
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.6.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -77,8 +77,8 @@
     assert repr(p) == ""
 
 def check_dir(p, expected):
-    got = set(name for name in dir(p) if not name.startswith('_'))
-    assert got == set(expected)
+    got = [name for name in dir(p) if not name.startswith('_')]
+    assert got == sorted(expected)
 
 def test_inspect_primitive_type():
     p = new_primitive_type("signed char")
@@ -3608,3 +3608,23 @@
     #
     py.test.raises(ValueError, unpack, p0, -1)
     py.test.raises(ValueError, unpack, p, -1)
+
+def test_cdata_dir():
+    BInt = new_primitive_type("int")
+    p = cast(BInt, 42)
+    check_dir(p, [])
+    p = newp(new_array_type(new_pointer_type(BInt), None), 5)
+    check_dir(p, [])
+    BStruct = new_struct_type("foo")
+    p = cast(new_pointer_type(BStruct), 0)
+    check_dir(p, [])    # opaque
+    complete_struct_or_union(BStruct, [('a2', BInt, -1),
+                                       ('a1', BInt, -1)])
+    check_dir(p, ['a1', 'a2'])   # always sorted
+    p = newp(new_pointer_type(BStruct), None)
+    check_dir(p, ['a1', 'a2'])
+    check_dir(p[0], ['a1', 'a2'])
+    pp = newp(new_pointer_type(new_pointer_type(BStruct)), p)
+    check_dir(pp, [])
+    check_dir(pp[0], ['a1', 'a2'])
+    check_dir(pp[0][0], ['a1', 'a2'])
diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py
--- a/pypy/module/_minimal_curses/test/test_curses.py
+++ b/pypy/module/_minimal_curses/test/test_curses.py
@@ -1,4 +1,4 @@
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.tool.udir import udir
 import py
 import sys
@@ -70,7 +70,7 @@
         f.write(source)
         child = self.spawn(['--withmod-_minimal_curses', str(f)])
         child.expect('ok!')
-        
+
 class TestCCurses(object):
     """ Test compiled version
     """
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -4,7 +4,7 @@
 
 import py
 
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import ll2ctypes
diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py
--- a/pypy/module/cpyext/bytearrayobject.py
+++ b/pypy/module/cpyext/bytearrayobject.py
@@ -85,7 +85,7 @@
     w_buffer = space.call_function(space.w_bytearray, w_obj)
     return make_ref(space, w_buffer)
 
- at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, result_is_ll=True)
+ at cpython_api([CONST_STRING, Py_ssize_t], PyObject, result_is_ll=True)
 def PyByteArray_FromStringAndSize(space, char_p, length):
     """Create a new bytearray object from string and its length, len.  On
     failure, NULL is returned."""
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.0-alpha0"
-#define PYPY_VERSION_NUM  0x05030000
+#define PYPY_VERSION "5.3.1-alpha0"
+#define PYPY_VERSION_NUM  0x05030100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer,
+    PyVarObject, Py_buffer, size_t,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -18,14 +18,14 @@
 PyBUF_SIMPLE = 0x0000
 PyBUF_WRITABLE = 0x0001
 
- at cpython_api([Py_ssize_t], rffi.VOIDP)
+ at cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
     return lltype.malloc(rffi.VOIDP.TO, size,
                          flavor='raw',
                          add_memory_pressure=True)
 
- at cpython_api([rffi.VOIDP, Py_ssize_t], rffi.VOIDP)
+ at cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
         return lltype.malloc(rffi.VOIDP.TO, size,
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -30,7 +30,8 @@
                  #endif
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
-                     printf("tp_basicsize==%ld\\n", s->ob_type->tp_basicsize);
+                     printf("tp_basicsize==%ld\\n",
+                            (long)s->ob_type->tp_basicsize); 
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -104,7 +105,7 @@
                  PyObject* s1 = PyByteArray_FromStringAndSize("test", 4);
                  if (s1 == NULL)
                      return NULL;
-                 char* c = PyByteArray_AsString(s1);
+                 const char* c = PyByteArray_AsString(s1);
                  PyObject* s2 = PyByteArray_FromStringAndSize(c, 4);
                  Py_DECREF(s1);
                  return s2;
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -4,7 +4,7 @@
 
 import py, pytest
 
-from pypy.conftest import pypydir
+from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -495,8 +495,8 @@
         c = array([1.e+110, 1.e-110], dtype=complex128)
         d = floor_divide(c**2, c)
         assert (d == [1.e+110, 0]).all()
-        
-        
+
+
 
     def test_basic(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -374,8 +374,8 @@
         a = np.array(data, dtype=b)
         x = pickle.loads(pickle.dumps(a))
         assert (x == a).all()
-        assert x.dtype == a.dtype 
-        
+        assert x.dtype == a.dtype
+
     def test_index(self):
         import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
@@ -1461,7 +1461,7 @@
                      "'offsets':[0,76800], "
                      "'itemsize':80000, "
                      "'aligned':True}")
-        
+
         assert dt == np.dtype(eval(str(dt)))
 
         dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1878,7 +1878,7 @@
         assert map(isnan, e) == [False, False, False, True, False]
         assert map(isinf, e) == [False, False, True, False, False]
         assert e.argmax() == 3
-        # numpy preserves value for uint16 -> cast_as_float16 -> 
+        # numpy preserves value for uint16 -> cast_as_float16 ->
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
         all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
@@ -2599,7 +2599,7 @@
         a = np.arange(6).reshape(2,3)
         i = np.dtype('int32').type(0)
         assert (a[0] == a[i]).all()
-        
+
 
     def test_ellipsis_indexing(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -200,7 +200,7 @@
         from numpy import arange, dtype
         from cPickle import loads, dumps
         import sys
-        
+
         a = arange(15).astype(object)
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, dumps, a)
@@ -211,4 +211,4 @@
         a = arange(15).astype(object).reshape((3, 5))
         b = loads(dumps(a))
         assert (a == b).all()
-        
+
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -20,6 +20,10 @@
         self.w_sockets = self.space.wrap([])
         if platform.machine().startswith('arm'):
             self.w_timeout = self.space.wrap(0.06)
+        if platform.machine().startswith('s390x'):
+            # s390x is not slow, but it seems there is one case when epoll
+            # modify method is called that takes longer on s390x
+            self.w_timeout = self.space.wrap(0.06)
         else:
             self.w_timeout = self.space.wrap(0.02)
 
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -2,7 +2,7 @@
 Implementation of interpreter-level 'sys' routines.
 """
 import os
-import pypy
+from pypy import pypydir
 
 # ____________________________________________________________
 #
@@ -20,7 +20,6 @@
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
         # Initialize the default path
-        pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
         srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.wrap_fsdecoded(p) for p in path])
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,11 +10,11 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 1, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
-pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
+pypydir = pypy.pypydir
 pypyroot = os.path.dirname(pypydir)
 del pypy
 from rpython.tool.version import get_repo_version_info
diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py
--- a/pypy/module/termios/test/test_termios.py
+++ b/pypy/module/termios/test/test_termios.py
@@ -1,7 +1,7 @@
 import os
 import sys
 import py
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.tool.udir import udir
 
 if os.name != 'posix':
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1359,8 +1359,8 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("enum foo;")
         from cffi import __version_info__
-        if __version_info__ < (1, 7):
-            py.test.skip("re-enable me in version 1.7")
+        if __version_info__ < (1, 8):
+            py.test.skip("re-enable me in version 1.8")
         e = py.test.raises(CDefError, ffi.cast, "enum foo", -1)
         assert str(e.value) == (
             "'enum foo' has no values explicitly defined: refusing to guess "
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1909,3 +1909,10 @@
     assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'],
                                 ['CFFIa', 'CFFIcc', 'CFFIccc'],
                                 ['CFFIaa', 'CFFIaaa', 'CFFIg'])
+
+def test_bool_in_cpp():
+    # this works when compiled as C, but in cffi < 1.7 it fails as C++
+    ffi = FFI()
+    ffi.cdef("bool f(void);")
+    lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
+    assert lib.f() == 1
diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py
--- a/pypy/objspace/test/test_binop_overriding.py
+++ b/pypy/objspace/test/test_binop_overriding.py
@@ -73,7 +73,7 @@
                 if C is not object:
                     setattr(C, name, f)
                 override_in_hier(n-1)
-                if C is not object:        
+                if C is not object:
                     delattr(C, name)
 
         override_in_hier()
@@ -105,7 +105,7 @@
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-         
+
         class X(Base):
             pass
         class Y(X):
@@ -116,7 +116,7 @@
         assert not fail
 
     def test_binop_combinations_sub(self):
-        Base, do_test = self.helpers        
+        Base, do_test = self.helpers
         class X(Base):
             pass
         class Y(X):
@@ -124,13 +124,13 @@
 
         fail = do_test(X, Y, 'sub', lambda x,y: x-y)
         #print len(fail)
-        assert not fail        
+        assert not fail
 
     def test_binop_combinations_pow(self):
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-        
+
         class X(Base):
             pass
         class Y(X):
@@ -138,13 +138,13 @@
 
         fail = do_test(X, Y, 'pow', lambda x,y: x**y)
         #print len(fail)
-        assert not fail        
+        assert not fail
 
     def test_binop_combinations_more_exhaustive(self):
         if not self.appdirect:
             skip("very slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-        
+
         class X(Base):
             pass
 
diff --git a/pypy/tool/genstatistic.py b/pypy/tool/genstatistic.py
--- a/pypy/tool/genstatistic.py
+++ b/pypy/tool/genstatistic.py
@@ -1,22 +1,22 @@
 
 import py
-from py._cmdline import pycountloc as countloc 
+from py._cmdline import pycountloc as countloc
 from py.xml import raw
-from pypy import conftest
+from pypy import pypydir
 
-pypydir = py.path.local(conftest.pypydir)
+pypydir = py.path.local(pypydir)
 
 def isdocfile(p):
     return (p.ext in ('.txt', '.rst') or
             p.basename in ('README', 'NOTES', 'LICENSE'))
 
 def istestfile(p):
-    if not p.check(file=1, ext='.py'): 
-        return False 
-    pb = p.purebasename 
-    if pb.startswith('test_') or pb.endswith('_test'): 
-        return True 
-    if 'test' in [x.basename for x in p.parts()[-4:]]: 
+    if not p.check(file=1, ext='.py'):
+        return False
+    pb = p.purebasename
+    if pb.startswith('test_') or pb.endswith('_test'):
+        return True
+    if 'test' in [x.basename for x in p.parts()[-4:]]:
         return True
 
 notistestfile = lambda x: not istestfile(x)
@@ -24,42 +24,43 @@
 class relchecker:
     def __init__(self, rel):
         self.rel = rel
-    def __call__(self, p): 
-        return p.relto(conftest.pypydir).startswith(self.rel)
+
+    def __call__(self, p):
+        return p.relto(pypydir).startswith(self.rel)
 
 def isfile(p):
     return p.check(file=1) and p.ext in ('.py', '.txt', '')
 
 def recpypy(p):
-    if p.basename[0] == '.': 
-        return False 
-    if p.basename in ('Pyrex', 
-                      '_cache', 
-                      'unicodedata', 
+    if p.basename[0] == '.':
+        return False
+    if p.basename in ('Pyrex',
+                      '_cache',
+                      'unicodedata',
                       'pypy-translation-snapshot'):
-        return False 
-    return True 
+        return False
+    return True
 
 def getpypycounter():
-    filecounter = countloc.FileCounter() 
-    root = py.path.local(conftest.pypydir)
+    filecounter = countloc.FileCounter()
+    root = py.path.local(pypydir)
     filecounter.addrecursive(root, isfile, rec=recpypy)
-    return filecounter 
+    return filecounter
 
-class CounterModel: 
-    def __init__(self, pypycounter): 
-        self.counter = pypycounter 
-        self.totallines = pypycounter.numlines 
+class CounterModel:
+    def __init__(self, pypycounter):
+        self.counter = pypycounter
+        self.totallines = pypycounter.numlines
         self.totalfiles = pypycounter.numfiles
-        self.testlines = pypycounter.getnumlines(istestfile) 
-        self.testfiles = pypycounter.getnumfiles(istestfile) 
-        self.notestlines = pypycounter.getnumlines(notistestfile) 
-        self.notestfiles = pypycounter.getnumfiles(notistestfile) 
+        self.testlines = pypycounter.getnumlines(istestfile)
+        self.testfiles = pypycounter.getnumfiles(istestfile)
+        self.notestlines = pypycounter.getnumlines(notistestfile)
+        self.notestfiles = pypycounter.getnumfiles(notistestfile)
         self.doclines = pypycounter.getnumlines(isdocfile)
-        self.docfiles = pypycounter.getnumfiles(isdocfile) 
+        self.docfiles = pypycounter.getnumfiles(isdocfile)
 
 #
-# rendering 
+# rendering
 #
 def row(*args):
     return html.tr([html.td(arg) for arg in args])
@@ -69,22 +70,22 @@
 
 def viewlocsummary(model):
     t = html.table(
-        row("total number of lines", model.totallines, raw(" ")), 
-        row("number of testlines", model.testlines, 
-            percent(model.testlines, model.totallines)), 
-        row("number of non-testlines", model.notestlines, 
-            percent(model.notestlines, model.totallines)), 
+        row("total number of lines", model.totallines, raw(" ")),
+        row("number of testlines", model.testlines,
+            percent(model.testlines, model.totallines)),
+        row("number of non-testlines", model.notestlines,
+            percent(model.notestlines, model.totallines)),
 
-        row("total number of files", model.totalfiles, raw(" ")), 
-        row("number of testfiles", model.testfiles, 
-            percent(model.testfiles, model.totalfiles)), 
-        row("number of non-testfiles", model.notestfiles, 
-            percent(model.notestfiles, model.totalfiles)), 
+        row("total number of files", model.totalfiles, raw(" ")),
+        row("number of testfiles", model.testfiles,
+            percent(model.testfiles, model.totalfiles)),
+        row("number of non-testfiles", model.notestfiles,
+            percent(model.notestfiles, model.totalfiles)),
         )
-    if model.docfiles: 
-        t.append(row("number of docfiles", model.docfiles, 
+    if model.docfiles:
+        t.append(row("number of docfiles", model.docfiles,
             percent(model.docfiles, model.totalfiles)))
-        t.append(row("number of doclines", model.doclines, 
+        t.append(row("number of doclines", model.doclines,
             percent(model.doclines, model.totallines)))
     return t
 
@@ -92,46 +93,46 @@
     t = html.table()
     d = model.counter.file2numlines
     paths = d.items()
-    paths.sort(lambda x,y : -cmp(x[1], y[1])) # sort by numlines 
-    for p, numlines in paths: 
-        if numlines < 3: 
+    paths.sort(lambda x, y: -cmp(x[1], y[1]))  # sort by numlines
+    for p, numlines in paths:
+        if numlines < 3:
             continue
         t.append(row(p.relto(pypydir.dirpath()), numlines))
     return t
 
-def viewsubdirs(model): 
+def viewsubdirs(model):
     t = html.table()
-    for p in pypydir.listdir(): 
-        if p.basename in '_cache .svn'.split(): 
+    for p in pypydir.listdir():
+        if p.basename in '_cache .svn'.split():
             continue
-        if p.check(dir=1): 
+        if p.check(dir=1):
             counter = countloc.FileCounter()
             counter.addrecursive(p, isfile, recpypy)
-            model = CounterModel(counter) 
+            model = CounterModel(counter)
             t.append(row(html.h2(p.relto(pypydir.dirpath()))))
             t.append(viewlocsummary(model))
             t.append(viewloclist(model))
     return t
 
-if __name__ == '__main__': 
+if __name__ == '__main__':
     if len(py.std.sys.argv) >= 2:
         target = py.path.local(py.std.sys.argv[1])
     else:
         target = py.path.local('index.html')
     print "writing source statistics to", target
-    pypycounter = getpypycounter() 
-    model = CounterModel(pypycounter) 
-    rev = py.path.svnwc(conftest.pypydir).info().rev
+    pypycounter = getpypycounter()
+    model = CounterModel(pypycounter)
+    rev = py.path.svnwc(pypydir).info().rev
     html = py.xml.html
     doc = html.html(
         html.head(
             html.title("PyPy Statistics %d" % rev),
-        ), 
+        ),
         html.body(
             html.h2("rev %d PyPy Summary of Files and Lines" % rev),
-            viewlocsummary(model), 
-            html.h2("Details on first-level subdirectories"), 
-            viewsubdirs(model), 
+            viewlocsummary(model),
+            html.h2("Details on first-level subdirectories"),
+            viewsubdirs(model),
             html.h3("PyPy Full List Files and Lines"),
             viewloclist(model),
             html.p("files with less than 3 lines ignored")
@@ -139,4 +140,3 @@
     )
     content = doc.unicode(indent=2).encode('utf8')
     target.write(content)
-
diff --git a/pypy/tool/getdocstrings.py b/pypy/tool/getdocstrings.py
--- a/pypy/tool/getdocstrings.py
+++ b/pypy/tool/getdocstrings.py
@@ -1,7 +1,7 @@
 import re
 from os import listdir
 from sys import stdin, stdout, stderr
-from pypy.conftest import pypydir
+from pypy import pypydir
 
 where = pypydir + '/objspace/std/'
 quote = '(' + "'" + '|' + '"' + ')'
@@ -29,7 +29,7 @@
 
 def compile_typedef(typ):
     return re.compile(r"(?P\s+)"
-                      + r"(?P" + typ 
+                      + r"(?P" + typ
                       + "_typedef = StdTypeDef+\s*\(\s*"
                       + quote + typ +  quote + ",).*"
                       + r"(?P^\s+)"
@@ -38,7 +38,7 @@
 
 def get_pypydoc(sourcefile):
     doc = compile_doc()
-    
+
     try: # if this works we already have a docstring
         pypydoc = doc.search(sourcefile).group('docstring')
 
@@ -86,14 +86,10 @@
 if __name__ == '__main__':
 
     filenames = mk_std_filelist()
-    
+
     for f in filenames:
         inf = file(where + f).read()
         outs = add_docstring(f[:-7], inf)
         if outs is not None:
             outf = file(where + f, 'w')
             outf.write(outs)
-        
-
-                
-            
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -5,9 +5,9 @@
 from pypy.interpreter.error import OperationError, oefmt
 
 try:
+    from _pytest.assertion.reinterpret import reinterpret as interpret
+except ImportError:
     from _pytest.assertion.newinterpret import interpret
-except ImportError:
-    from _pytest.assertion.oldinterpret import interpret
 
 # ____________________________________________________________
 
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,15 +1,16 @@
 # Edit these appropriately before running this script
 maj=5
-min=1
-rev=2
+min=3
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
 echo checking hg log -r $tagname
 hg log -r $tagname || exit 1

From pypy.commits at gmail.com  Wed Jun  8 14:59:56 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 11:59:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57586b2c.089d1c0a.da538.485d@mx.google.com>

Author: Ronan Lamy 
Branch: py3.5
Changeset: r85041:038f4cc08cb1
Date: 2016-06-08 19:59 +0100
http://bitbucket.org/pypy/pypy/changeset/038f4cc08cb1/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 40436 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,7 @@
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,17 +104,20 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas Stührk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -140,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +175,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,11 +210,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -228,7 +230,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -270,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +297,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/TODO b/TODO
deleted file mode 100644
--- a/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-* reduce size of generated c code from slot definitions in slotdefs.
-* remove broken DEBUG_REFCOUNT from pyobject.py
diff --git a/dotviewer/graphserver.py b/dotviewer/graphserver.py
--- a/dotviewer/graphserver.py
+++ b/dotviewer/graphserver.py
@@ -143,6 +143,11 @@
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
+        if len(sys.argv) == 1:
+            # start locally
+            import sshgraphserver
+            sshgraphserver.ssh_graph_server(['LOCAL'])
+            sys.exit(0)
         print >> sys.stderr, __doc__
         sys.exit(2)
     if sys.argv[1] == '--stdio':
diff --git a/dotviewer/sshgraphserver.py b/dotviewer/sshgraphserver.py
--- a/dotviewer/sshgraphserver.py
+++ b/dotviewer/sshgraphserver.py
@@ -4,11 +4,14 @@
 
 Usage:
     sshgraphserver.py  hostname  [more args for ssh...]
+    sshgraphserver.py  LOCAL
 
 This logs in to 'hostname' by passing the arguments on the command-line
 to ssh.  No further configuration is required: it works for all programs
 using the dotviewer library as long as they run on 'hostname' under the
 same username as the one sshgraphserver logs as.
+
+If 'hostname' is the string 'LOCAL', then it starts locally without ssh.
 """
 
 import graphserver, socket, subprocess, random
@@ -18,12 +21,19 @@
     s1 = socket.socket()
     s1.bind(('127.0.0.1', socket.INADDR_ANY))
     localhost, localport = s1.getsockname()
-    remoteport = random.randrange(10000, 20000)
-    #  ^^^ and just hope there is no conflict
 
-    args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)]
-    args = args + sshargs + ['python -u -c "exec input()"']
-    print ' '.join(args[:-1])
+    if sshargs[0] != 'LOCAL':
+        remoteport = random.randrange(10000, 20000)
+        #  ^^^ and just hope there is no conflict
+
+        args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (
+            remoteport, localport)]
+        args = args + sshargs + ['python -u -c "exec input()"']
+    else:
+        remoteport = localport
+        args = ['python', '-u', '-c', 'exec input()']
+
+    print ' '.join(args)
     p = subprocess.Popen(args, bufsize=0,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE)
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -1735,7 +1735,6 @@
             ("__reversed__", reversed, empty_seq, set(), {}),
             ("__length_hint__", list, zero, set(),
              {"__iter__" : iden, "next" : stop}),
-            ("__sizeof__", sys.getsizeof, zero, set(), {}),
             ("__instancecheck__", do_isinstance, return_true, set(), {}),
             ("__missing__", do_dict_missing, some_number,
              set(("__class__",)), {}),
@@ -1747,6 +1746,8 @@
             ("__format__", format, format_impl, set(), {}),
             ("__dir__", dir, empty_seq, set(), {}),
             ]
+        if test_support.check_impl_detail():
+            specials.append(("__sizeof__", sys.getsizeof, zero, set(), {}))
 
         class Checker(object):
             def __getattr__(self, attr, test=self):
@@ -1768,10 +1769,6 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
-            if name == '__length_hint__' or name == '__sizeof__':
-                if not test_support.check_impl_detail():
-                    continue
-
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -328,8 +328,8 @@
 
     def test_13_genexp(self):
         if self.using_gc:
+            gc.enable()
             test_support.gc_collect()
-            gc.enable()
         try:
             self.run_test(generator_example)
             # issue1265: if the trace function contains a generator,
diff --git a/lib-python/3/ctypes/test/test_python_api.py b/lib-python/3/ctypes/test/test_python_api.py
--- a/lib-python/3/ctypes/test/test_python_api.py
+++ b/lib-python/3/ctypes/test/test_python_api.py
@@ -18,6 +18,7 @@
 
 class PythonAPITestCase(unittest.TestCase):
 
+    @xfail
     def test_PyBytes_FromStringAndSize(self):
         PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize
 
@@ -67,6 +68,7 @@
         del pyobj
         self.assertEqual(grc(s), ref)
 
+    @xfail
     def test_PyOS_snprintf(self):
         PyOS_snprintf = pythonapi.PyOS_snprintf
         PyOS_snprintf.argtypes = POINTER(c_char), c_size_t, c_char_p
@@ -81,6 +83,7 @@
         # not enough arguments
         self.assertRaises(TypeError, PyOS_snprintf, buf)
 
+    @xfail
     def test_pyobject_repr(self):
         self.assertEqual(repr(py_object()), "py_object()")
         self.assertEqual(repr(py_object(42)), "py_object(42)")
diff --git a/lib-python/3/ensurepip/__init__.py b/lib-python/3/ensurepip/__init__.py
--- a/lib-python/3/ensurepip/__init__.py
+++ b/lib-python/3/ensurepip/__init__.py
@@ -8,9 +8,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "18.2"
+_SETUPTOOLS_VERSION = "21.2.1"
 
-_PIP_VERSION = "7.1.2"
+_PIP_VERSION = "8.1.2"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/3/ensurepip/_bundled/pip-8.1.2-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-8.1.2-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cc49227a0c7e13757f4863a9b7ace1eb56c3ce61
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/setuptools-21.2.1-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-21.2.1-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..fe36464f79ba87960c33f3bdff817deb9e4e5f7c
GIT binary patch

[cut]

diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py
--- a/lib-python/3/inspect.py
+++ b/lib-python/3/inspect.py
@@ -1666,6 +1666,7 @@
                             _ClassMethodWrapper,
                             types.BuiltinFunctionType)
 
+_builtin_code_type = type(dict.update.__code__)
 
 def _signature_get_user_defined_method(cls, method_name):
     """Private helper. Checks if ``cls`` has an attribute
@@ -1677,7 +1678,14 @@
     except AttributeError:
         return
     else:
-        if not isinstance(meth, _NonUserDefinedCallables):
+        # The particular check cpython uses to determine if a particular method
+        # is a builtin or not doesn't work on pypy. The following code is
+        # pypy-specific.
+        try:
+            code = meth.__code__
+        except AttributeError:
+            return
+        if not isinstance(code, _builtin_code_type):
             # Once '__signature__' will be added to 'C'-level
             # callables, this check won't be necessary
             return meth
diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py
--- a/lib-python/3/subprocess.py
+++ b/lib-python/3/subprocess.py
@@ -1124,15 +1124,18 @@
             c2pread, c2pwrite = -1, -1
             errread, errwrite = -1, -1
 
+            ispread = False
             if stdin is None:
                 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _winapi.CreatePipe(None, 0)
                     p2cread = Handle(p2cread)
                     _winapi.CloseHandle(_)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
                 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
+                ispread = True
             elif stdin == DEVNULL:
                 p2cread = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdin, int):
@@ -1140,17 +1143,20 @@
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _winapi.CreatePipe(None, 0)
                     c2pwrite = Handle(c2pwrite)
                     _winapi.CloseHandle(_)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
                 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
+                ispwrite = True
             elif stdout == DEVNULL:
                 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdout, int):
@@ -1158,17 +1164,20 @@
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _winapi.CreatePipe(None, 0)
                     errwrite = Handle(errwrite)
                     _winapi.CloseHandle(_)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _winapi.CreatePipe(None, 0)
                 errread, errwrite = Handle(errread), Handle(errwrite)
+                ispwrite = True
             elif stderr == STDOUT:
                 errwrite = c2pwrite
             elif stderr == DEVNULL:
@@ -1178,19 +1187,23 @@
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
 
             return (p2cread, p2cwrite,
                     c2pread, c2pwrite,
                     errread, errwrite)
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             h = _winapi.DuplicateHandle(
                 _winapi.GetCurrentProcess(), handle,
                 _winapi.GetCurrentProcess(), 0, 1,
                 _winapi.DUPLICATE_SAME_ACCESS)
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
+                handle.Close()
             return Handle(h)
 
 
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -41,6 +41,16 @@
         'scripts': '{base}/bin',
         'data': '{base}',
         },
+    'pypy': {
+        'stdlib': '{installed_base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{installed_base}/include',
+        'platinclude': '{installed_base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{installed_base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -171,7 +181,9 @@
 
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
diff --git a/lib-python/3/test/pickletester.py b/lib-python/3/test/pickletester.py
--- a/lib-python/3/test/pickletester.py
+++ b/lib-python/3/test/pickletester.py
@@ -13,7 +13,7 @@
 
 from test.support import (
     TestFailed, TESTFN, run_with_locale, no_tracing,
-    _2G, _4G, bigmemtest, check_impl_detail
+    _2G, _4G, bigmemtest, check_impl_detail, impl_detail
     )
 
 from pickle import bytes_types
@@ -1781,6 +1781,7 @@
                 loaded = self.loads(dumped)
                 self.assert_is_copy(obj, loaded)
 
+    @impl_detail("pypy does not store attribute names", pypy=False)
     def test_attribute_name_interning(self):
         # Test that attribute names of pickled objects are interned when
         # unpickling.
@@ -1795,6 +1796,7 @@
             for x_key, y_key in zip(x_keys, y_keys):
                 self.assertIs(x_key, y_key)
 
+    @impl_detail("This test is too strong indeed", pypy=False)
     def test_pickle_to_2x(self):
         # Pickle non-trivial data with protocol 2, expecting that it yields
         # the same result as Python 2.x did.
diff --git a/lib-python/3/test/test_cmd_line_script.py b/lib-python/3/test/test_cmd_line_script.py
--- a/lib-python/3/test/test_cmd_line_script.py
+++ b/lib-python/3/test/test_cmd_line_script.py
@@ -42,7 +42,11 @@
 _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
 print('__loader__==%a' % _loader)
 print('__file__==%a' % __file__)
-print('__cached__==%a' % __cached__)
+if __cached__ is not None:
+    # XXX: test_script_compiled on PyPy
+    assertEqual(__file__, __cached__)
+    if not __cached__.endswith(('pyc', 'pyo')):
+        raise AssertionError('has __cached__ but not compiled')
 print('__package__==%r' % __package__)
 # Check PEP 451 details
 import os.path
@@ -224,8 +228,9 @@
     def test_basic_script(self):
         with support.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
+            package = '' if support.check_impl_detail(pypy=True) else None
             self._check_script(script_name, script_name, script_name,
-                               script_dir, None,
+                               script_dir, package,
                                importlib.machinery.SourceFileLoader)
 
     def test_script_compiled(self):
@@ -234,8 +239,9 @@
             py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             pyc_file = support.make_legacy_pyc(script_name)
+            package = '' if support.check_impl_detail(pypy=True) else None
             self._check_script(pyc_file, pyc_file,
-                               pyc_file, script_dir, None,
+                               pyc_file, script_dir, package,
                                importlib.machinery.SourcelessFileLoader)
 
     def test_directory(self):
diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -1840,7 +1840,6 @@
             ("__reversed__", reversed, empty_seq, set(), {}),
             ("__length_hint__", list, zero, set(),
              {"__iter__" : iden, "__next__" : stop}),
-            ("__sizeof__", sys.getsizeof, zero, set(), {}),
             ("__instancecheck__", do_isinstance, return_true, set(), {}),
             ("__missing__", do_dict_missing, some_number,
              set(("__class__",)), {}),
@@ -1857,6 +1856,8 @@
             ("__dir__", dir, empty_seq, set(), {}),
             ("__round__", round, zero, set(), {}),
             ]
+        if support.check_impl_detail():
+            specials.append(("__sizeof__", sys.getsizeof, zero, set(), {}))
 
         class Checker(object):
             def __getattr__(self, attr, test=self):
@@ -2019,7 +2020,8 @@
         except TypeError as msg:
             self.assertIn("weak reference", str(msg))
         else:
-            self.fail("weakref.ref(no) should be illegal")
+            if support.check_impl_detail(pypy=False):
+                self.fail("weakref.ref(no) should be illegal")
         class Weak(object):
             __slots__ = ['foo', '__weakref__']
         yes = Weak()
@@ -4213,14 +4215,10 @@
         self.assertNotEqual(l.__add__, [5].__add__)
         self.assertNotEqual(l.__add__, l.__mul__)
         self.assertEqual(l.__add__.__name__, '__add__')
-        if hasattr(l.__add__, '__self__'):
+        self.assertIs(l.__add__.__self__, l)
+        if hasattr(l.__add__, '__objclass__'):
             # CPython
-            self.assertIs(l.__add__.__self__, l)
             self.assertIs(l.__add__.__objclass__, list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertIs(l.__add__.im_self, l)
-            self.assertIs(l.__add__.im_class, list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
             hash(l.__add__)
@@ -4430,9 +4428,9 @@
         with self.assertRaises(TypeError) as cm:
             type(list).__dict__["__doc__"].__set__(list, "blah")
         self.assertIn("can't set list.__doc__", str(cm.exception))
-        with self.assertRaises(TypeError) as cm:
+        with self.assertRaises((AttributeError, TypeError)) as cm:
             type(X).__dict__["__doc__"].__delete__(X)
-        self.assertIn("can't delete X.__doc__", str(cm.exception))
+        self.assertIn("delete", str(cm.exception))
         self.assertEqual(X.__doc__, "banana")
 
     def test_qualname(self):
@@ -4441,9 +4439,16 @@
 
         # make sure we have an example of each type of descriptor
         for d, n in zip(descriptors, types):
+            if (support.check_impl_detail(pypy=True) and
+                n in ('method', 'member', 'wrapper')):
+                # PyPy doesn't have these
+                continue
             self.assertEqual(type(d).__name__, n + '_descriptor')
 
         for d in descriptors:
+            if (support.check_impl_detail(pypy=True) and
+                not hasattr(d, '__objclass__')):
+                continue
             qualname = d.__objclass__.__qualname__ + '.' + d.__name__
             self.assertEqual(d.__qualname__, qualname)
 
@@ -4454,7 +4459,7 @@
 
         class X:
             pass
-        with self.assertRaises(TypeError):
+        with self.assertRaises((AttributeError, TypeError)):
             del X.__qualname__
 
         self.assertRaises(TypeError, type.__dict__['__qualname__'].__set__,
@@ -4492,6 +4497,8 @@
         for o in gc.get_objects():
             self.assertIsNot(type(o), X)
 
+    @unittest.skipIf(support.check_impl_detail(pypy=True),
+                     "https://bitbucket.org/pypy/pypy/issues/2306")
     def test_object_new_and_init_with_parameters(self):
         # See issue #1683368
         class OverrideNeither:
@@ -4659,6 +4666,7 @@
 
 
 class MiscTests(unittest.TestCase):
+    @support.cpython_only
     def test_type_lookup_mro_reference(self):
         # Issue #14199: _PyType_Lookup() has to keep a strong reference to
         # the type MRO because it may be modified during the lookup, if
diff --git a/lib-python/3/test/test_ensurepip.py b/lib-python/3/test/test_ensurepip.py
--- a/lib-python/3/test/test_ensurepip.py
+++ b/lib-python/3/test/test_ensurepip.py
@@ -310,7 +310,7 @@
 
     @requires_usable_pip
     def test_bootstrap_version(self):
-        with test.support.captured_stdout() as stdout:
+        with test.support.captured_stderr() as stdout:
             with self.assertRaises(SystemExit):
                 ensurepip._main(["--version"])
         result = stdout.getvalue().strip()
@@ -335,7 +335,7 @@
 class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase):
 
     def test_uninstall_version(self):
-        with test.support.captured_stdout() as stdout:
+        with test.support.captured_stderr() as stdout:
             with self.assertRaises(SystemExit):
                 ensurepip._uninstall._main(["--version"])
         result = stdout.getvalue().strip()
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -158,11 +158,12 @@
             self.assertEqual(cm.exception.lineno, lineno)
             self.assertEqual(cm.exception.offset, offset)
 
+        is_pypy = check_impl_detail(pypy=True)
         check('def fact(x):\n\treturn x!\n', 2, 10)
-        check('1 +\n', 1, 4)
-        check('def spam():\n  print(1)\n print(2)', 3, 10)
-        check('Python = "Python" +', 1, 20)
-        check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
+        check('1 +\n', 1, 4 - is_pypy)
+        check('def spam():\n  print(1)\n print(2)', 3, 0 if is_pypy else 10)
+        check('Python = "Python" +', 1, 20 - is_pypy)
+        check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy)
 
     @cpython_only
     def testSettingException(self):
@@ -417,10 +418,11 @@
             self.fail("No exception raised")
 
     def testInvalidAttrs(self):
+        delerrs = (AttributeError, TypeError)
         self.assertRaises(TypeError, setattr, Exception(), '__cause__', 1)
-        self.assertRaises(TypeError, delattr, Exception(), '__cause__')
+        self.assertRaises(delerrs, delattr, Exception(), '__cause__')
         self.assertRaises(TypeError, setattr, Exception(), '__context__', 1)
-        self.assertRaises(TypeError, delattr, Exception(), '__context__')
+        self.assertRaises(delerrs, delattr, Exception(), '__context__')
 
     def testNoneClearsTracebackAttr(self):
         try:
diff --git a/lib-python/3/test/test_site.py b/lib-python/3/test/test_site.py
--- a/lib-python/3/test/test_site.py
+++ b/lib-python/3/test/test_site.py
@@ -6,7 +6,8 @@
 """
 import unittest
 import test.support
-from test.support import captured_stderr, TESTFN, EnvironmentVarGuard
+from test.support import (
+    captured_stderr, check_impl_detail, TESTFN, EnvironmentVarGuard)
 import builtins
 import os
 import sys
@@ -230,7 +231,11 @@
         site.PREFIXES = ['xoxo']
         dirs = site.getsitepackages()
 
-        if (sys.platform == "darwin" and
+        if check_impl_detail(pypy=True):
+            self.assertEqual(len(dirs), 1)
+            wanted = os.path.join('xoxo', 'site-packages')
+            self.assertEqual(dirs[0], wanted)
+        elif (sys.platform == "darwin" and
             sysconfig.get_config_var("PYTHONFRAMEWORK")):
             # OS X framework builds
             site.PREFIXES = ['Python.framework']
@@ -346,8 +351,10 @@
 
         self.assertEqual(proc.returncode, 0)
         os__file__, os__cached__ = stdout.splitlines()[:2]
-        self.assertFalse(os.path.isabs(os__file__))
-        self.assertFalse(os.path.isabs(os__cached__))
+        if check_impl_detail(cpython=True):
+            # XXX: should probably match cpython
+            self.assertFalse(os.path.isabs(os__file__))
+            self.assertFalse(os.path.isabs(os__cached__))
         # Now, with 'import site', it works.
         proc = subprocess.Popen([sys.executable, '-c', command],
                                 env=env,
diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py
--- a/lib-python/3/test/test_socket.py
+++ b/lib-python/3/test/test_socket.py
@@ -748,10 +748,11 @@
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo')
-        self.assertIn('(1 given)', str(cm.exception))
+        if support.check_impl_detail():
+            self.assertIn(' given)', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 0, sockname, 4)
-        self.assertIn('(4 given)', str(cm.exception))
+        self.assertIn(' given', str(cm.exception))
 
     def testCrucialConstants(self):
         # Testing for mission critical constants
diff --git a/lib-python/3/test/test_sys_settrace.py b/lib-python/3/test/test_sys_settrace.py
--- a/lib-python/3/test/test_sys_settrace.py
+++ b/lib-python/3/test/test_sys_settrace.py
@@ -330,8 +330,8 @@
 
     def test_13_genexp(self):
         if self.using_gc:
+            gc.enable()
             support.gc_collect()
-            gc.enable()
         try:
             self.run_test(generator_example)
             # issue1265: if the trace function contains a generator,
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -6,7 +6,8 @@
 from copy import copy
 
 from test.support import (run_unittest, TESTFN, unlink, check_warnings,
-                          captured_stdout, skip_unless_symlink, change_cwd)
+                          captured_stdout, impl_detail, import_module,
+                          skip_unless_symlink, change_cwd)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -235,7 +236,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     @skip_unless_symlink
@@ -288,6 +289,7 @@
             _main()
         self.assertTrue(len(output.getvalue().split('\n')) > 0)
 
+    @impl_detail("PyPy lacks LDFLAGS/LDSHARED config vars", pypy=False)
     @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
     def test_ldshared_value(self):
         ldflags = sysconfig.get_config_var('LDFLAGS')
@@ -338,6 +340,7 @@
         self.assertEqual(status, 0)
         self.assertEqual(my_platform, test_platform)
 
+    @impl_detail("Test is not PyPy compatible", pypy=False)
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')
@@ -407,6 +410,7 @@
 
 class MakefileTests(unittest.TestCase):
 
+    @impl_detail("Test is not PyPy compatible", pypy=False)
     @unittest.skipIf(sys.platform.startswith('win'),
                      'Test is not Windows compatible')
     def test_get_makefile_filename(self):
diff --git a/lib-python/3/test/test_tempfile.py b/lib-python/3/test/test_tempfile.py
--- a/lib-python/3/test/test_tempfile.py
+++ b/lib-python/3/test/test_tempfile.py
@@ -1347,7 +1347,6 @@
                          "were deleted")
         d2.cleanup()
 
-    @support.cpython_only
     def test_del_on_collection(self):
         # A TemporaryDirectory is deleted when garbage collected
         dir = tempfile.mkdtemp()
@@ -1355,6 +1354,7 @@
             d = self.do_create(dir=dir)
             name = d.name
             del d # Rely on refcounting to invoke __del__
+            support.gc_collect()
             self.assertFalse(os.path.exists(name),
                         "TemporaryDirectory %s exists after __del__" % name)
         finally:
diff --git a/lib-python/3/test/test_threading.py b/lib-python/3/test/test_threading.py
--- a/lib-python/3/test/test_threading.py
+++ b/lib-python/3/test/test_threading.py
@@ -466,11 +466,16 @@
     def test_is_alive_after_fork(self):
         # Try hard to trigger #18418: is_alive() could sometimes be True on
         # threads that vanished after a fork.
-        old_interval = sys.getswitchinterval()
-        self.addCleanup(sys.setswitchinterval, old_interval)
+        newgil = hasattr(sys, 'getswitchinterval')
+        if newgil:
+            geti, seti = sys.getswitchinterval, sys.setswitchinterval
+        else:
+            geti, seti = sys.getcheckinterval, sys.setcheckinterval
+        old_interval = geti()
+        self.addCleanup(seti, old_interval)
 
         # Make the bug more likely to manifest.
-        sys.setswitchinterval(1e-6)
+        seti(1e-6 if newgil else 1)
 
         for i in range(20):
             t = threading.Thread(target=lambda: None)
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -149,7 +149,7 @@
     RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'),
     RegrTest('test_codecmaps_kr.py', usemodules='_multibytecodec'),
     RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
-    RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'),
+    RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec struct unicodedata array'),
     RegrTest('test_codeop.py', core=True),
     RegrTest('test_coding.py', core=True),
     RegrTest('test_collections.py', usemodules='binascii struct'),
@@ -179,7 +179,7 @@
     RegrTest('test_decimal.py'),
     RegrTest('test_decorators.py', core=True),
     RegrTest('test_defaultdict.py', usemodules='_collections'),
-    RegrTest('test_deque.py', core=True, usemodules='_collections'),
+    RegrTest('test_deque.py', core=True, usemodules='_collections struct'),
     RegrTest('test_descr.py', core=True, usemodules='_weakref'),
     RegrTest('test_descrtut.py', core=True),
     RegrTest('test_devpoll.py'),
@@ -196,6 +196,7 @@
     RegrTest('test_dummy_threading.py', core=True),
     RegrTest('test_dynamic.py'),
     RegrTest('test_email', skip="XXX is a directory"),
+    RegrTest('test_ensurepip.py'),
     RegrTest('test_enumerate.py', core=True),
     RegrTest('test_eof.py', core=True),
     RegrTest('test_epoll.py'),
@@ -256,7 +257,7 @@
     RegrTest('test_importhooks.py', core=True),
     RegrTest('test_importlib', 'XXX is a directory'),
     RegrTest('test_index.py'),
-    RegrTest('test_inspect.py'),
+    RegrTest('test_inspect.py', usemodules="struct unicodedata"),
     RegrTest('test_int.py', core=True),
     RegrTest('test_int_literal.py', core=True),
     RegrTest('test_io.py', core=True, usemodules='array binascii'),
@@ -417,7 +418,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True),
+    RegrTest('test_time.py', core=True, usemodules="struct"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -384,6 +384,7 @@
         return self
 
 class defaultdict(dict):
+    __slots__ = ["default_factory"]
     
     def __init__(self, *args, **kwds):
         if len(args) > 0:
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -161,6 +161,15 @@
 _codecs.register_error('_decimal_encode', _handle_decimaldigits)
 
 
+def _unsafe_check(name, lo, hi, value):
+    if not -_sys.maxsize-1 <= value <= _sys.maxsize:
+        raise OverflowError(
+            "Python int too large to convert to C ssize_t")
+    if not lo <= value <= hi:
+        raise ValueError("valid range for unsafe %s is [%d, %d]" %
+                         (name, lo, hi))
+
+
 # Decimal class
 
 _DEC_MINALLOC = 4
@@ -298,7 +307,8 @@
                 raise ValueError("exponent must be an integer")
             if not -_sys.maxsize-1 <= exponent <= _sys.maxsize:
                 # Compatibility with CPython
-                raise OverflowError()
+                raise OverflowError(
+                    "Python int too large to convert to C ssize_t")
 
         # coefficients
         if not digits and not is_special:
@@ -1501,6 +1511,19 @@
             _mpdec.mpd_free(output)
         return result.decode()
 
+    if _sys.maxsize < 2**63-1:
+        def _unsafe_setprec(self, value):
+            _unsafe_check('prec', 1, 1070000000, value)
+            self._ctx.prec = value
+
+        def _unsafe_setemin(self, value):
+            _unsafe_check('emin', -1070000000, 0, value)
+            self._ctx.emin = value
+
+        def _unsafe_setemax(self, value):
+            _unsafe_check('emax', 0, 1070000000, value)
+            self._ctx.emax = value
+
 
 class _SignalDict(_collections.abc.MutableMapping):
 
diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vccompat.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2008-2016 Stefan Krah. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#ifndef VCCOMPAT_H
+#define VCCOMPAT_H
+
+
+/* Visual C fixes: no stdint.h, no snprintf ... */
+#ifdef _MSC_VER
+  #include "vcstdint.h"
+  #undef inline
+  #define inline __inline
+  #undef random
+  #define random rand
+  #undef srandom
+  #define srandom srand
+  #undef snprintf
+  #define snprintf sprintf_s
+  #define HAVE_SNPRINTF
+  #undef strncasecmp
+  #define strncasecmp _strnicmp
+  #undef strcasecmp
+  #define strcasecmp _stricmp
+  #undef strtoll
+  #define strtoll _strtoi64
+  #define strdup _strdup
+  #define PRIi64 "I64i"
+  #define PRIu64 "I64u"
+  #define PRIi32 "I32i"
+  #define PRIu32 "I32u"
+#endif
+
+
+#endif /* VCCOMPAT_H */
+
+
+
diff --git a/lib_pypy/_libmpdec/vcdiv64.asm b/lib_pypy/_libmpdec/vcdiv64.asm
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vcdiv64.asm
@@ -0,0 +1,48 @@
+;
+; Copyright (c) 2008-2016 Stefan Krah. All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+;
+; 1. Redistributions of source code must retain the above copyright
+;    notice, this list of conditions and the following disclaimer.
+;
+; 2. Redistributions in binary form must reproduce the above copyright
+;    notice, this list of conditions and the following disclaimer in the
+;    documentation and/or other materials provided with the distribution.
+;
+; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+; ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+; SUCH DAMAGE.
+;
+
+
+PUBLIC    _mpd_div_words
+_TEXT    SEGMENT
+q$ = 8
+r$ = 16
+hi$ = 24
+lo$ = 32
+d$ = 40
+_mpd_div_words PROC
+    mov    r10, rdx
+    mov    rdx, r8
+    mov    rax, r9
+    div    QWORD PTR d$[rsp]
+    mov    QWORD PTR [r10], rdx
+    mov    QWORD PTR [rcx], rax
+    ret    0
+_mpd_div_words ENDP
+_TEXT    ENDS
+END
+
+
diff --git a/lib_pypy/_libmpdec/vcstdint.h b/lib_pypy/_libmpdec/vcstdint.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vcstdint.h
@@ -0,0 +1,232 @@
+// ISO C9x  compliant stdint.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
+//
+//  Copyright (c) 2006-2008 Alexander Chemeris
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   1. Redistributions of source code must retain the above copyright notice,
+//      this list of conditions and the following disclaimer.
+//
+//   2. Redistributions in binary form must reproduce the above copyright
+//      notice, this list of conditions and the following disclaimer in the
+//      documentation and/or other materials provided with the distribution.
+//
+//   3. The name of the author may be used to endorse or promote products
+//      derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_STDINT_H_ // [
+#define _MSC_STDINT_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include 
+
+// For Visual Studio 6 in C++ mode wrap  include with 'extern "C++" {}'
+// or compiler give many errors like this:
+//   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
+#if (_MSC_VER < 1300) && defined(__cplusplus)
+   extern "C++" {
+#endif
+#     include 
+#if (_MSC_VER < 1300) && defined(__cplusplus)
+   }
+#endif
+
+// Define _W64 macros to mark types changing their size, like intptr_t.
+#ifndef _W64
+#  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
+#     define _W64 __w64
+#  else
+#     define _W64
+#  endif
+#endif
+
+
+// 7.18.1 Integer types
+
+// 7.18.1.1 Exact-width integer types
+typedef __int8            int8_t;
+typedef __int16           int16_t;
+typedef __int32           int32_t;
+typedef __int64           int64_t;
+typedef unsigned __int8   uint8_t;
+typedef unsigned __int16  uint16_t;
+typedef unsigned __int32  uint32_t;
+typedef unsigned __int64  uint64_t;
+
+// 7.18.1.2 Minimum-width integer types
+typedef int8_t    int_least8_t;
+typedef int16_t   int_least16_t;
+typedef int32_t   int_least32_t;
+typedef int64_t   int_least64_t;
+typedef uint8_t   uint_least8_t;
+typedef uint16_t  uint_least16_t;
+typedef uint32_t  uint_least32_t;
+typedef uint64_t  uint_least64_t;
+
+// 7.18.1.3 Fastest minimum-width integer types
+typedef int8_t    int_fast8_t;
+typedef int16_t   int_fast16_t;
+typedef int32_t   int_fast32_t;
+typedef int64_t   int_fast64_t;
+typedef uint8_t   uint_fast8_t;
+typedef uint16_t  uint_fast16_t;
+typedef uint32_t  uint_fast32_t;
+typedef uint64_t  uint_fast64_t;
+
+// 7.18.1.4 Integer types capable of holding object pointers
+#ifdef _WIN64 // [
+   typedef __int64           intptr_t;
+   typedef unsigned __int64  uintptr_t;
+#else // _WIN64 ][
+   typedef _W64 int               intptr_t;
+   typedef _W64 unsigned int      uintptr_t;
+#endif // _WIN64 ]
+
+// 7.18.1.5 Greatest-width integer types
+typedef int64_t   intmax_t;
+typedef uint64_t  uintmax_t;
+
+
+// 7.18.2 Limits of specified-width integer types
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
+
+// 7.18.2.1 Limits of exact-width integer types
+#define INT8_MIN     ((int8_t)_I8_MIN)
+#define INT8_MAX     _I8_MAX
+#define INT16_MIN    ((int16_t)_I16_MIN)
+#define INT16_MAX    _I16_MAX
+#define INT32_MIN    ((int32_t)_I32_MIN)
+#define INT32_MAX    _I32_MAX
+#define INT64_MIN    ((int64_t)_I64_MIN)
+#define INT64_MAX    _I64_MAX
+#define UINT8_MAX    _UI8_MAX
+#define UINT16_MAX   _UI16_MAX
+#define UINT32_MAX   _UI32_MAX
+#define UINT64_MAX   _UI64_MAX
+
+// 7.18.2.2 Limits of minimum-width integer types
+#define INT_LEAST8_MIN    INT8_MIN
+#define INT_LEAST8_MAX    INT8_MAX
+#define INT_LEAST16_MIN   INT16_MIN
+#define INT_LEAST16_MAX   INT16_MAX
+#define INT_LEAST32_MIN   INT32_MIN
+#define INT_LEAST32_MAX   INT32_MAX
+#define INT_LEAST64_MIN   INT64_MIN
+#define INT_LEAST64_MAX   INT64_MAX
+#define UINT_LEAST8_MAX   UINT8_MAX
+#define UINT_LEAST16_MAX  UINT16_MAX
+#define UINT_LEAST32_MAX  UINT32_MAX
+#define UINT_LEAST64_MAX  UINT64_MAX
+
+// 7.18.2.3 Limits of fastest minimum-width integer types
+#define INT_FAST8_MIN    INT8_MIN
+#define INT_FAST8_MAX    INT8_MAX
+#define INT_FAST16_MIN   INT16_MIN
+#define INT_FAST16_MAX   INT16_MAX
+#define INT_FAST32_MIN   INT32_MIN
+#define INT_FAST32_MAX   INT32_MAX
+#define INT_FAST64_MIN   INT64_MIN
+#define INT_FAST64_MAX   INT64_MAX
+#define UINT_FAST8_MAX   UINT8_MAX
+#define UINT_FAST16_MAX  UINT16_MAX
+#define UINT_FAST32_MAX  UINT32_MAX
+#define UINT_FAST64_MAX  UINT64_MAX
+
+// 7.18.2.4 Limits of integer types capable of holding object pointers
+#ifdef _WIN64 // [
+#  define INTPTR_MIN   INT64_MIN
+#  define INTPTR_MAX   INT64_MAX
+#  define UINTPTR_MAX  UINT64_MAX
+#else // _WIN64 ][
+#  define INTPTR_MIN   INT32_MIN
+#  define INTPTR_MAX   INT32_MAX
+#  define UINTPTR_MAX  UINT32_MAX
+#endif // _WIN64 ]
+
+// 7.18.2.5 Limits of greatest-width integer types
+#define INTMAX_MIN   INT64_MIN
+#define INTMAX_MAX   INT64_MAX
+#define UINTMAX_MAX  UINT64_MAX
+
+// 7.18.3 Limits of other integer types
+
+#ifdef _WIN64 // [
+#  define PTRDIFF_MIN  _I64_MIN
+#  define PTRDIFF_MAX  _I64_MAX
+#else  // _WIN64 ][
+#  define PTRDIFF_MIN  _I32_MIN
+#  define PTRDIFF_MAX  _I32_MAX
+#endif  // _WIN64 ]
+
+#define SIG_ATOMIC_MIN  INT_MIN
+#define SIG_ATOMIC_MAX  INT_MAX
+
+#ifndef SIZE_MAX // [
+#  ifdef _WIN64 // [
+#     define SIZE_MAX  _UI64_MAX
+#  else // _WIN64 ][
+#     define SIZE_MAX  _UI32_MAX
+#  endif // _WIN64 ]
+#endif // SIZE_MAX ]
+
+// WCHAR_MIN and WCHAR_MAX are also defined in 
+#ifndef WCHAR_MIN // [
+#  define WCHAR_MIN  0
+#endif  // WCHAR_MIN ]
+#ifndef WCHAR_MAX // [
+#  define WCHAR_MAX  _UI16_MAX
+#endif  // WCHAR_MAX ]
+
+#define WINT_MIN  0
+#define WINT_MAX  _UI16_MAX
+
+#endif // __STDC_LIMIT_MACROS ]
+
+
+// 7.18.4 Limits of other integer types
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
+
+// 7.18.4.1 Macros for minimum-width integer constants
+
+#define INT8_C(val)  val##i8
+#define INT16_C(val) val##i16
+#define INT32_C(val) val##i32
+#define INT64_C(val) val##i64
+
+#define UINT8_C(val)  val##ui8
+#define UINT16_C(val) val##ui16
+#define UINT32_C(val) val##ui32
+#define UINT64_C(val) val##ui64
+
+// 7.18.4.2 Macros for greatest-width integer constants
+#define INTMAX_C   INT64_C
+#define UINTMAX_C  UINT64_C
+
+#endif // __STDC_CONSTANT_MACROS ]
+
+
+#endif // _MSC_STDINT_H_ ]
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat 
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
deleted file mode 100644
--- a/lib_pypy/_subprocess.py
+++ /dev/null
@@ -1,214 +0,0 @@
-"""
-Support routines for subprocess module.
-Currently, this extension module is only required when using the
-subprocess module on Windows.
-"""
-
-
-# Declare external Win32 functions
-
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
-
-GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
-
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessW
-_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
-
-# Now the _subprocess module implementation
-
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
-
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
-
-    def __int__(self):
-        return self.handle
-
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
-
-    def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
-
-    def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
-
-def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
-
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
-
-    if not res:
-        raise _WinError()
-
-    return _handle(read.value), _handle(write.value)
-
-def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
-
-def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
-
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
-
-    if not res:
-        raise _WinError()
-
-    return _handle(target.value)
-
-def CreateProcess(name, command_line, process_attr, thread_attr,
-                  inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
-    if startup_info is not None:
-        si.dwFlags = startup_info.dwFlags
-        si.wShowWindow = startup_info.wShowWindow
-        if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
-        if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
-        if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
-
-    pi = _PROCESS_INFORMATION()
-    flags |= CREATE_UNICODE_ENVIRONMENT
-
-    if env is not None:
-        envbuf = ""
-        for k, v in env.items():
-            envbuf += "%s=%s\0" % (k, v)
-        envbuf += '\0'
-    else:
-        envbuf = None
-
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
-
-    if not res:
-        raise _WinError()
-
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
-
-def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
-    if res < 0:
-        raise _WinError()
-
-    return res
-
-def GetExitCodeProcess(handle):
-    code = _c_int()
-
-    res = _GetExitCodeProcess(int(handle), _byref(code))
-
-    if not res:
-        raise _WinError()
-
-    return code.value
-
-def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
-
-    if not res:
-        raise _WinError()
-
-def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
-
-    if not res:
-        return None
-    else:
-        return res
-
-STD_INPUT_HANDLE = -10
-STD_OUTPUT_HANDLE = -11
-STD_ERROR_HANDLE = -12
-DUPLICATE_SAME_ACCESS = 2
-STARTF_USESTDHANDLES = 0x100
-STARTF_USESHOWWINDOW = 0x001
-SW_HIDE = 0
-INFINITE = 0xffffffff
-WAIT_OBJECT_0 = 0
-CREATE_NEW_CONSOLE = 0x010
-CREATE_NEW_PROCESS_GROUP = 0x200
-CREATE_UNICODE_ENVIRONMENT = 0x400
-STILL_ACTIVE = 259
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_winapi.py
@@ -0,0 +1,237 @@
+"""
+Support routines for subprocess module.
+Currently, this extension module is only required when using the
+subprocess module on Windows.
+"""
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
+
+# Declare external Win32 functions
+
+import ctypes
+
+_kernel32 = ctypes.WinDLL('kernel32')
+
+_CloseHandle = _kernel32.CloseHandle
+_CloseHandle.argtypes = [ctypes.c_int]
+_CloseHandle.restype = ctypes.c_int
+
+_CreatePipe = _kernel32.CreatePipe
+_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
+                        ctypes.c_void_p, ctypes.c_int]
+_CreatePipe.restype = ctypes.c_int
+
+_GetCurrentProcess = _kernel32.GetCurrentProcess
+_GetCurrentProcess.argtypes = []
+_GetCurrentProcess.restype = ctypes.c_int
+
+GetVersion = _kernel32.GetVersion
+GetVersion.argtypes = []
+GetVersion.restype = ctypes.c_int
+
+_DuplicateHandle = _kernel32.DuplicateHandle
+_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
+                             ctypes.POINTER(ctypes.c_int),
+                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
+_DuplicateHandle.restype = ctypes.c_int
+
+_WaitForSingleObject = _kernel32.WaitForSingleObject
+_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
+_WaitForSingleObject.restype = ctypes.c_int
+
+_GetExitCodeProcess = _kernel32.GetExitCodeProcess
+_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+_GetExitCodeProcess.restype = ctypes.c_int
+
+_TerminateProcess = _kernel32.TerminateProcess
+_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
+_TerminateProcess.restype = ctypes.c_int
+
+_GetStdHandle = _kernel32.GetStdHandle
+_GetStdHandle.argtypes = [ctypes.c_int]
+_GetStdHandle.restype = ctypes.c_int
+
+_GetModuleFileNameW = _kernel32.GetModuleFileNameW
+_GetModuleFileNameW.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_uint]
+_GetModuleFileNameW.restype = ctypes.c_int
+
+class _STARTUPINFO(ctypes.Structure):
+    _fields_ = [('cb',         ctypes.c_int),
+                ('lpReserved', ctypes.c_void_p),
+                ('lpDesktop',  ctypes.c_char_p),
+                ('lpTitle',    ctypes.c_char_p),
+                ('dwX',        ctypes.c_int),
+                ('dwY',        ctypes.c_int),
+                ('dwXSize',    ctypes.c_int),
+                ('dwYSize',    ctypes.c_int),
+                ('dwXCountChars', ctypes.c_int),
+                ('dwYCountChars', ctypes.c_int),
+                ("dwFillAttribute", ctypes.c_int),
+                ("dwFlags", ctypes.c_int),
+                ("wShowWindow", ctypes.c_short),
+                ("cbReserved2", ctypes.c_short),
+                ("lpReserved2", ctypes.c_void_p),
+                ("hStdInput", ctypes.c_int),
+                ("hStdOutput", ctypes.c_int),
+                ("hStdError", ctypes.c_int)
+                ]
+
+class _PROCESS_INFORMATION(ctypes.Structure):
+    _fields_ = [("hProcess", ctypes.c_int),
+                ("hThread", ctypes.c_int),
+                ("dwProcessID", ctypes.c_int),
+                ("dwThreadID", ctypes.c_int)]
+
+_CreateProcess = _kernel32.CreateProcessW
+_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
+                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
+                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
+_CreateProcess.restype = ctypes.c_int
+
+del ctypes
+
+# Now the _winapi module implementation
+
+from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+
+class _handle:
+    def __init__(self, handle):
+        self.handle = handle
+
+    def __int__(self):
+        return self.handle
+
+    def __del__(self):
+        if self.handle is not None:
+            _CloseHandle(self.handle)
+
+    def Detach(self):
+        handle, self.handle = self.handle, None
+        return handle
+
+    def Close(self):
+        if self.handle not in (-1, None):
+            _CloseHandle(self.handle)
+            self.handle = None
+
+def CreatePipe(attributes, size):
+    read = _c_int()
+    write = _c_int()
+
+    res = _CreatePipe(_byref(read), _byref(write), None, size)
+
+    if not res:
+        raise _WinError()
+
+    return _handle(read.value), _handle(write.value)
+
+def GetCurrentProcess():
+    return _handle(_GetCurrentProcess())
+
+def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
+    target = _c_int()
+
+    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
+                           _byref(target),
+                           access, inherit, options)
+
+    if not res:
+        raise _WinError()
+
+    return _handle(target.value)
+
+def CreateProcess(name, command_line, process_attr, thread_attr,
+                  inherit, flags, env, start_dir, startup_info):
+    si = _STARTUPINFO()
+    if startup_info is not None:
+        si.dwFlags = startup_info.dwFlags
+        si.wShowWindow = startup_info.wShowWindow
+        if startup_info.hStdInput:
+            si.hStdInput = int(startup_info.hStdInput)
+        if startup_info.hStdOutput:
+            si.hStdOutput = int(startup_info.hStdOutput)
+        if startup_info.hStdError:
+            si.hStdError = int(startup_info.hStdError)
+
+    pi = _PROCESS_INFORMATION()
+    flags |= CREATE_UNICODE_ENVIRONMENT
+
+    if env is not None:
+        envbuf = ""
+        for k, v in env.items():
+            envbuf += "%s=%s\0" % (k, v)
+        envbuf += '\0'
+    else:
+        envbuf = None
+
+    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
+                        start_dir, _byref(si), _byref(pi))
+
+    if not res:
+        raise _WinError()
+
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+
+def WaitForSingleObject(handle, milliseconds):
+    res = _WaitForSingleObject(int(handle), milliseconds)
+
+    if res < 0:
+        raise _WinError()
+
+    return res
+
+def GetExitCodeProcess(handle):
+    code = _c_int()
+
+    res = _GetExitCodeProcess(int(handle), _byref(code))
+
+    if not res:
+        raise _WinError()
+
+    return code.value
+
+def TerminateProcess(handle, exitcode):
+    res = _TerminateProcess(int(handle), exitcode)
+
+    if not res:
+        raise _WinError()
+
+def GetStdHandle(stdhandle):
+    res = _GetStdHandle(stdhandle)
+
+    if not res:
+        return None
+    else:
+        return res
+
+def CloseHandle(handle):
+    res = _CloseHandle(handle)
+
+    if not res:
+        raise _WinError()
+
+def GetModuleFileName(module):
+    buf = ctypes.create_unicode_buffer(_MAX_PATH)
+    res = _GetModuleFileNameW(module, buf, _MAX_PATH)
+
+    if not res:
+        raise _WinError()
+    return buf.value
+
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+STD_ERROR_HANDLE = -12
+DUPLICATE_SAME_ACCESS = 2
+STARTF_USESTDHANDLES = 0x100
+STARTF_USESHOWWINDOW = 0x001
+SW_HIDE = 0
+INFINITE = 0xffffffff
+WAIT_OBJECT_0 = 0
+WAIT_TIMEOUT = 0x102
+CREATE_NEW_CONSOLE = 0x010
+CREATE_NEW_PROCESS_GROUP = 0x200
+CREATE_UNICODE_ENVIRONMENT = 0x400
+STILL_ACTIVE = 259
+_MAX_PATH = 260
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -332,8 +332,8 @@
     def from_buffer(self, python_buffer):
         """Return a  that points to the data of the
         given Python object, which must support the buffer interface.
-        Note that this is not meant to be used on the built-in types str,
-        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        Note that this is not meant to be used on the built-in types
+        str or unicode (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
         """
@@ -397,20 +397,7 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gcp = self._backend.gcp
-        except AttributeError:
-            pass
-        else:
-            return gcp(cdata, destructor)
-        #
-        with self._lock:
-            try:
-                gc_weakrefs = self.gc_weakrefs
-            except AttributeError:
-                from .gc_weakref import GcWeakrefs
-                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-            return gc_weakrefs.build(cdata, destructor)
+        return self._backend.gcp(cdata, destructor)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):

From pypy.commits at gmail.com  Wed Jun  8 15:54:37 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 12:54:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix after the merge
Message-ID: <575877fd.d21b1c0a.e68e.61bc@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85042:3ef853b2c167
Date: 2016-06-08 20:54 +0100
http://bitbucket.org/pypy/pypy/changeset/3ef853b2c167/

Log:	fix after the merge

diff --git a/pypy/module/posix/test/test_interp_posix.py b/pypy/module/posix/test/test_interp_posix.py
--- a/pypy/module/posix/test/test_interp_posix.py
+++ b/pypy/module/posix/test/test_interp_posix.py
@@ -3,7 +3,7 @@
 import py
 
 from rpython.tool.udir import udir
-from pypy.conftest import pypydir
+from pypy import pypydir
 from pypy.module.posix.interp_posix import convert_seconds
 
 class TestPexpect(object):
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -16,7 +16,8 @@
 from pypy.tool.pytest import appsupport
 from pypy.tool.pytest.objspace import gettestobjspace
 from rpython.tool.udir import udir
-from pypy.conftest import PyPyClassCollector, pypydir
+from pypy import pypydir
+from pypy.conftest import PyPyClassCollector
 from inspect import getmro
 
 pypyroot = os.path.dirname(pypydir)
@@ -58,7 +59,7 @@
         else:
             return r
     elif isinstance(value, type):
-        return type.__name__    
+        return type.__name__
     else:
         return repr(value)
 

From pypy.commits at gmail.com  Wed Jun  8 16:37:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 13:37:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: Rename the RPython helpers from
 ll_int_floordiv() to ll_int_py_div(), 
Message-ID: <57588212.8b371c0a.ae99c.5bfa@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85043:fbfec7254a35
Date: 2016-06-08 21:26 +0200
http://bitbucket.org/pypy/pypy/changeset/fbfec7254a35/

Log:	Rename the RPython helpers from ll_int_floordiv() to
	ll_int_py_div(), because they are doing Python-style rounding,
	unlike llop.int_floordiv().

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -23,7 +23,7 @@
             guard_true(i14, descr=...)
             guard_not_invalidated(descr=...)
             i16 = int_eq(i6, %d)
-            i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=)
+            i19 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i6, i10, descr=)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
             i22 = int_ge(i19, i10)
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -252,7 +252,7 @@
 def _ll_2_int_floordiv(x, y):
     # this is used only if the RPython program uses llop.int_floordiv()
     # explicitly.  For 'a // b', see _handle_int_special() in jtransform.py.
-    # This is the reverse of rpython.rtyper.rint.ll_int_floordiv(), i.e.
+    # This is the reverse of rpython.rtyper.rint.ll_int_py_div(), i.e.
     # the same logic as rpython.rtyper.lltypesystem.opimpl.op_int_floordiv
     # but written in a no-branch style.
     r = x // y
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -478,7 +478,7 @@
             except ZeroDivisionError:
                 return -42
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_floordiv_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_div_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
@@ -505,7 +505,7 @@
                 return 42
         # XXX so far, this really produces a int_mod_ovf_zer...
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -236,11 +236,11 @@
         return _rtype_template(hop, 'mul_ovf')
 
     def rtype_floordiv(_, hop):
-        return _rtype_call_helper(hop, 'floordiv', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div', [ZeroDivisionError])
     rtype_inplace_floordiv = rtype_floordiv
 
     def rtype_floordiv_ovf(_, hop):
-        return _rtype_call_helper(hop, 'floordiv_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div_ovf', [ZeroDivisionError])
 
     # turn 'div' on integers into 'floordiv'
     rtype_div         = rtype_floordiv
@@ -250,11 +250,11 @@
     # 'def rtype_truediv' is delegated to the superclass FloatRepr
 
     def rtype_mod(_, hop):
-        return _rtype_call_helper(hop, 'mod', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod', [ZeroDivisionError])
     rtype_inplace_mod = rtype_mod
 
     def rtype_mod_ovf(_, hop):
-        return _rtype_call_helper(hop, 'mod_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod_ovf', [ZeroDivisionError])
 
     def rtype_xor(_, hop):
         return _rtype_template(hop, 'xor')
@@ -319,7 +319,7 @@
     vlist = hop.inputargs(repr, repr2)
     prefix = repr.opprefix
 
-    if '_ovf' in func or func.startswith(('mod', 'floordiv')):
+    if '_ovf' in func or func.startswith(('py_mod', 'py_div')):
         if prefix+func not in ('int_add_ovf', 'int_add_nonneg_ovf',
                                'int_sub_ovf', 'int_mul_ovf'):
             raise TyperError("%r should not be used here any more" % (func,))
@@ -353,7 +353,7 @@
             any_implicit_exception = True
 
     if not any_implicit_exception:
-        if not func.startswith(('mod', 'floordiv')):
+        if not func.startswith(('py_mod', 'py_div')):
             return _rtype_template(hop, func)
 
     repr = hop.r_result
@@ -388,7 +388,7 @@
 # ---------- floordiv ----------
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv(x, y):
+def ll_int_py_div(x, y):
     # Python, and RPython, assume that integer division truncates
     # towards -infinity.  However, in C, integer division truncates
     # towards 0.  So assuming that, we need to apply a correction
@@ -400,159 +400,159 @@
     return r + (u >> INT_BITS_1)
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv_nonnegargs(x, y):
+def ll_int_py_div_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_floordiv(Signed, x, y)            # <= truncates like in C
-    ll_assert(r >= 0, "int_floordiv_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_div_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_floordiv_zer(x, y):
+def ll_int_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf(x, y):
+def ll_int_py_div_ovf(x, y):
     # JIT: intentionally not short-circuited to produce only one guard
     # and to remove the check fully if one of the arguments is known
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf_zer(x, y):
+def ll_int_py_div_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv_ovf(x, y)
+    return ll_int_py_div_ovf(x, y)
 
 @jit.oopspec("int.udiv(x, y)")
-def ll_uint_floordiv(x, y):
+def ll_uint_py_div(x, y):
     return llop.uint_floordiv(Unsigned, x, y)
 
-def ll_uint_floordiv_zer(x, y):
+def ll_uint_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("unsigned integer division")
-    return ll_uint_floordiv(x, y)
+    return ll_uint_py_div(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_floordiv      = ll_int_floordiv
-    ll_llong_floordiv_zer  = ll_int_floordiv_zer
-    ll_ullong_floordiv     = ll_uint_floordiv
-    ll_ullong_floordiv_zer = ll_uint_floordiv_zer
+    ll_llong_py_div      = ll_int_py_div
+    ll_llong_py_div_zer  = ll_int_py_div_zer
+    ll_ullong_py_div     = ll_uint_py_div
+    ll_ullong_py_div_zer = ll_uint_py_div_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_floordiv(x, y):
+    def ll_llong_py_div(x, y):
         r = llop.llong_floordiv(SignedLongLong, x, y)  # <= truncates like in C
         p = r * y
         if y < 0: u = p - x
         else:     u = x - p
         return r + (u >> LLONG_BITS_1)
 
-    def ll_llong_floordiv_zer(x, y):
+    def ll_llong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("longlong division")
-        return ll_llong_floordiv(x, y)
+        return ll_llong_py_div(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_floordiv(x, y):
+    def ll_ullong_py_div(x, y):
         return llop.ullong_floordiv(UnsignedLongLong, x, y)
 
-    def ll_ullong_floordiv_zer(x, y):
+    def ll_ullong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("unsigned longlong division")
-        return ll_ullong_floordiv(x, y)
+        return ll_ullong_py_div(x, y)
 
 @jit.dont_look_inside
-def ll_lllong_floordiv(x, y):
+def ll_lllong_py_div(x, y):
     r = llop.lllong_floordiv(SignedLongLongLong, x, y) # <= truncates like in C
     p = r * y
     if y < 0: u = p - x
     else:     u = x - p
     return r + (u >> LLLONG_BITS_1)
 
-def ll_lllong_floordiv_zer(x, y):
+def ll_lllong_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("longlonglong division")
-    return ll_lllong_floordiv(x, y)
+    return ll_lllong_py_div(x, y)
 
 
 # ---------- mod ----------
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod(x, y):
+def ll_int_py_mod(x, y):
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> INT_BITS_1))
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod_nonnegargs(x, y):
+def ll_int_py_mod_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
-    ll_assert(r >= 0, "int_mod_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_mod_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_mod_zer(x, y):
+def ll_int_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf(x, y):
-    # see comment in ll_int_floordiv_ovf
+def ll_int_py_mod_ovf(x, y):
+    # see comment in ll_int_py_div_ovf
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf_zer(x, y):
+def ll_int_py_mod_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod_ovf(x, y)
+    return ll_int_py_mod_ovf(x, y)
 
 @jit.oopspec("int.umod(x, y)")
-def ll_uint_mod(x, y):
+def ll_uint_py_mod(x, y):
     return llop.uint_mod(Unsigned, x, y)
 
-def ll_uint_mod_zer(x, y):
+def ll_uint_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_uint_mod(x, y)
+    return ll_uint_py_mod(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_mod      = ll_int_mod
-    ll_llong_mod_zer  = ll_int_mod_zer
-    ll_ullong_mod     = ll_uint_mod
-    ll_ullong_mod_zer = ll_uint_mod_zer
+    ll_llong_py_mod      = ll_int_py_mod
+    ll_llong_py_mod_zer  = ll_int_py_mod_zer
+    ll_ullong_py_mod     = ll_uint_py_mod
+    ll_ullong_py_mod_zer = ll_uint_py_mod_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_mod(x, y):
+    def ll_llong_py_mod(x, y):
         r = llop.llong_mod(SignedLongLong, x, y)    # <= truncates like in C
         if y < 0: u = -r
         else:     u = r
         return r + (y & (u >> LLONG_BITS_1))
 
-    def ll_llong_mod_zer(x, y):
+    def ll_llong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return ll_llong_mod(x, y)
+        return ll_llong_py_mod(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_mod(x, y):
+    def ll_ullong_py_mod(x, y):
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
-    def ll_ullong_mod_zer(x, y):
+    def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
 @jit.dont_look_inside
-def ll_lllong_mod(x, y):
+def ll_lllong_py_mod(x, y):
     r = llop.lllong_mod(SignedLongLongLong, x, y)  # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> LLLONG_BITS_1))
 
-def ll_lllong_mod_zer(x, y):
+def ll_lllong_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_lllong_mod(x, y)
+    return ll_lllong_py_mod(x, y)
 
 
 # ---------- lshift, neg, abs ----------
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -390,7 +390,7 @@
         res = self.interpret(f, [sys.maxint])
         assert res == 0
 
-    def test_int_floordiv_nonnegargs(self):
+    def test_int_py_div_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
@@ -398,7 +398,7 @@
         res = self.interpret(f, [1234567, 123])
         assert res == 1234567 // 123
 
-    def test_int_mod_nonnegargs(self):
+    def test_int_py_mod_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py
--- a/rpython/translator/test/test_simplify.py
+++ b/rpython/translator/test/test_simplify.py
@@ -55,7 +55,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \
@@ -73,7 +73,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \

From pypy.commits at gmail.com  Wed Jun  8 16:45:32 2016
From: pypy.commits at gmail.com (stefanor)
Date: Wed, 08 Jun 2016 13:45:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Reject mkdir() in read-only sandbox
 filesystems
Message-ID: <575883ec.6150c20a.7783c.3cad@mx.google.com>

Author: Stefano Rivera 
Branch: 
Changeset: r85044:0513b4a90c28
Date: 2016-06-08 22:39 +0200
http://bitbucket.org/pypy/pypy/changeset/0513b4a90c28/

Log:	Reject mkdir() in read-only sandbox filesystems

diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py
--- a/rpython/translator/sandbox/sandlib.py
+++ b/rpython/translator/sandbox/sandlib.py
@@ -530,6 +530,9 @@
     def do_ll_os__ll_os_unlink(self, vpathname):
         raise OSError(errno.EPERM, "write access denied")
 
+    def do_ll_os__ll_os_mkdir(self, vpathname, mode=None):
+        raise OSError(errno.EPERM, "write access denied")
+
     def do_ll_os__ll_os_getuid(self):
         return UID
     do_ll_os__ll_os_geteuid = do_ll_os__ll_os_getuid

From pypy.commits at gmail.com  Wed Jun  8 17:26:40 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 08 Jun 2016 14:26:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use full Unpack Grammar,
 one more call argument fix
Message-ID: <57588d90.0654c20a.1573.4adf@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85045:b7332e2d0f64
Date: 2016-06-08 23:25 +0200
http://bitbucket.org/pypy/pypy/changeset/b7332e2d0f64/

Log:	Use full Unpack Grammar, one more call argument fix

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1085,8 +1085,7 @@
             args = None
         if not keywords:
             keywords = None
-        return ast.Call(callable_expr, args, keywords, variable_arg,
-                        keywords_arg, callable_expr.lineno,
+        return ast.Call(callable_expr, args, keywords, callable_expr.lineno,
                         callable_expr.col_offset)
 
     def parse_number(self, raw):
diff --git a/pypy/interpreter/pyparser/data/Grammar3.5 b/pypy/interpreter/pyparser/data/Grammar3.5
--- a/pypy/interpreter/pyparser/data/Grammar3.5
+++ b/pypy/interpreter/pyparser/data/Grammar3.5
@@ -128,10 +128,10 @@
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
-arglist: (argument ',')* (argument [',']
-                         |'*' test (',' argument)* [',' '**' test] 
-                         |'**' test)
-#arglist: argument (',' argument)*  [',']
+#arglist: (argument ',')* (argument [',']
+#                         |'*' test (',' argument)* [',' '**' test] 
+#                         |'**' test)
+arglist: argument (',' argument)*  [',']
 
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
@@ -142,11 +142,11 @@
 # Illegal combinations and orderings are blocked in ast.c:
 # multiple (test comp_for) arguements are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
-#argument: ( test [comp_for] |
-#            test '=' test |
-#            '**' test |
-#            '*' test )
+#argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+argument: ( test [comp_for] |
+            test '=' test |
+            '**' test |
+            '*' test )
 
 comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist 'in' or_test [comp_iter]

From pypy.commits at gmail.com  Wed Jun  8 19:39:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Jun 2016 16:39:10 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Remove irrelevant test: with -A,
 all tests are app-level, so the applevel marker is useless
Message-ID: <5758ac9e.03c31c0a.1ef99.ffff9378@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85046:95a6de756261
Date: 2016-06-09 00:38 +0100
http://bitbucket.org/pypy/pypy/changeset/95a6de756261/

Log:	Remove irrelevant test: with -A, all tests are app-level, so the
	applevel marker is useless

diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -20,11 +20,3 @@
         assert not skipped and not failed
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
-
-    def test_appdirect(self, testdir):
-        sorter = testdir.inline_run(innertest, '-m', 'applevel', '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
-        assert len(passed) == 2
-        print passed
-        assert "app_test_something" in passed[0].nodeid
-        assert "test_method_app" in passed[1].nodeid

From pypy.commits at gmail.com  Thu Jun  9 02:35:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 23:35:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #266: I think it would show up
 here, so better change the assert()
Message-ID: <57590e3e.c99d1c0a.f9746.fffffcd1@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r2718:50807f7805db
Date: 2016-06-09 08:37 +0200
http://bitbucket.org/cffi/cffi/changeset/50807f7805db/

Log:	Issue #266: I think it would show up here, so better change the
	assert() into a real test that runs also on non-debugging
	compilation.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5258,7 +5258,20 @@
                         "libffi failed to build this callback");
         goto error;
     }
-    assert(closure->user_data == infotuple);
+    if (closure->user_data != infotuple) {
+        /* Issue #266.  Should not occur, but could, if we are using
+           at runtime a version of libffi compiled with a different
+           'ffi_closure' structure than the one we expect from ffi.h
+           (e.g. difference in details of the platform): a difference
+           in FFI_TRAMPOLINE_SIZE means that the 'user_data' field
+           ends up somewhere else, and so the test above fails.
+        */
+        PyErr_SetString(PyExc_SystemError,
+            "ffi_prep_closure(): bad user_data (it seems that the "
+            "version of the libffi library seen at runtime is "
+            "different from the 'ffi.h' file seen at compile-time)");
+        goto error;
+    }
     return (PyObject *)cd;
 
  error:

From pypy.commits at gmail.com  Thu Jun  9 02:37:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 23:37:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/50807f7805db
Message-ID: <57590e90.c7b81c0a.71767.fffff1da@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85047:1b5056abe827
Date: 2016-06-09 08:37 +0200
http://bitbucket.org/pypy/pypy/changeset/1b5056abe827/

Log:	update to cffi/50807f7805db

diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -439,7 +439,7 @@
     prepare_int_lshift = helper.prepare_binary_op
     prepare_int_rshift = helper.prepare_binary_op
     prepare_uint_rshift = helper.prepare_binary_op
-    prepare_uint_mul_high = helper.prepare_int_mul_ovf
+    prepare_uint_mul_high = helper.prepare_binary_op
 
     prepare_int_add_ovf = helper.prepare_binary_op
     prepare_int_sub_ovf = helper.prepare_binary_op
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -148,7 +148,8 @@
                                                  ('elements', FFI_TYPE_PP)])
 
     ffi_cif = rffi_platform.Struct('ffi_cif', [])
-    ffi_closure = rffi_platform.Struct('ffi_closure', [])
+    ffi_closure = rffi_platform.Struct('ffi_closure',
+                                       [('user_data', rffi.VOIDP)])
 
 def add_simple_type(type_name):
     for name in ['size', 'alignment', 'type']:

From pypy.commits at gmail.com  Thu Jun  9 02:43:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 23:43:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: extra_args is a list,
 indexing it starts at 0..?
Message-ID: <57591019.441ac20a.67e79.ffff9ce3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85048:5c70dc3c49be
Date: 2016-06-09 08:44 +0200
http://bitbucket.org/pypy/pypy/changeset/5c70dc3c49be/

Log:	extra_args is a list, indexing it starts at 0..?

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -295,8 +295,8 @@
     options = parser.parse_args(args)
 
     # Handle positional arguments, choke if both methods are used
-    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
-                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
+    for i,target, default in ([0, 'name', ''], [1, 'pypy_c', pypy_exe],
+                              [2, 'targetdir', ''], [3,'override_pypy_c', '']):
         if len(options.extra_args)>i:
             if getattr(options, target) != default:
                 print 'positional argument',i,target,'already has value',getattr(options, target)

From pypy.commits at gmail.com  Thu Jun  9 02:49:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Jun 2016 23:49:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test for 5c70dc3c49be (the
 pypy-root argument was required but ignored)
Message-ID: <57591163.e7c9c20a.9dfba.ffffd2ee@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85049:0188cbebef64
Date: 2016-06-09 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/0188cbebef64/

Log:	fix test for 5c70dc3c49be (the pypy-root argument was required but
	ignored)

diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -21,7 +21,7 @@
 
     def test_dir_structure(self, test='test'):
         retval, builddir = package.package(
-            '--without-cffi', str(py.path.local(pypydir).dirpath()),
+            '--without-cffi',
             test, self.rename_pypy_c, _fake=True)
         assert retval == 0
         prefix = builddir.join(test)

From pypy.commits at gmail.com  Thu Jun  9 03:23:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Jun 2016 00:23:49 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: arg. applied change to wrong file
Message-ID: <57591985.264bc20a.5f60f.ffffe109@mx.google.com>

Author: Richard Plangger 
Branch: release-5.x
Changeset: r85050:3d0db4a026b8
Date: 2016-06-09 09:23 +0200
http://bitbucket.org/pypy/pypy/changeset/3d0db4a026b8/

Log:	arg. applied change to wrong file

diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -439,7 +439,7 @@
     prepare_int_lshift = helper.prepare_binary_op
     prepare_int_rshift = helper.prepare_binary_op
     prepare_uint_rshift = helper.prepare_binary_op
-    prepare_uint_mul_high = helper.prepare_int_mul_ovf
+    prepare_uint_mul_high = helper.prepare_binary_op
 
     prepare_int_add_ovf = helper.prepare_binary_op
     prepare_int_sub_ovf = helper.prepare_binary_op

From pypy.commits at gmail.com  Thu Jun  9 03:36:53 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Thu, 09 Jun 2016 00:36:53 -0700 (PDT)
Subject: [pypy-commit] pypy gc-forkfriendly: Fixing little to nothing,
 make incminimark_remoteheader more correctish, maybe.
Message-ID: <57591c95.4275c20a.63b1e.ffffe9d1@mx.google.com>

Author: Devin Jeanpierre 
Branch: gc-forkfriendly
Changeset: r85051:79a9cecb91c5
Date: 2016-06-09 00:35 -0700
http://bitbucket.org/pypy/pypy/changeset/79a9cecb91c5/

Log:	Fixing little to nothing, make incminimark_remoteheader more
	correctish, maybe.

	e.g. test_zrpy_gc, these two fail (with an assert failure and
	segfault resp.):

	 TestRemoteHeaderShadowStack.test_compile_framework_7_interior
	TestRemoteHeaderShadowStack.test_compile_framework_9

	Some rambling notes on the above tests:

	* test_compile_framework_7_interior fails with an assert error:

	 PyPy assertion failed at rpython_memory_gc.c:11879: in
	pypy_g_IncrementalMiniMarkGCBase__collect_obj: non-pinned nursery
	obj in _collect_obj

	 I find this really confusing and disturbing. Nothing I do to
	nursery objects should be any different. In fact, I tried changing
	set_flags/add_flags/etc. to check is_in_nursery, like this:

	 if self.is_in_nursery(obj): return
	incminimark.IncrementalMiniMarkGCBase.add_flags(self, obj, flags)

	 Even this didn't solve it, even though it meant that every
	operation was on the tid field for nursery objects, and should be
	identical to incminimark!

	 If I unconditionally called
	incminimark.IncrementalMiniMarkGCBase.add_flags etc., then it only
	passed tests if I did it for *all three* mutating methods.

	 My conclusion is that the remote_flags for some object in the
	nursery is ignored, and we were setting flags on it at some point
	even though it was is_in_nursery reported it was not in the nursery.
	(e.g.: maybe it was copied to the nursery, but the remote_flags
	field wasn't included. But why?)

	 Maybe it's time to step through in rr/gdb again.

	* test_compile_framework_9 fails with a segfault.

	 I think this is purely because I have not found all the ways this
	interacts poorly with the JIT compiler, and so it is solvable. For
	example, it is fixed if I do the "if self.is_in_nursery(obj):"
	checks above. My guess is that somewhere, nursery objects are
	allocated with a garbage remote_flags (like the one in the JIT I
	think I fixed in this commit), and I just need to catch that case,
	or else completely stop using remote_flags for young objects.

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -146,6 +146,7 @@
     round_up              = False
     write_barrier_descr   = None
     fielddescr_tid        = None
+    fielddescr_remote_flags  = None
     gcrootmap             = None
     str_type_id           = 0
     unicode_type_id       = 0
@@ -393,6 +394,10 @@
 
     def _setup_tid(self):
         self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid')
+        if self.GCClass.has_remote_flags:
+            self.fielddescr_remote_flags = get_field_descr(self, self.GCClass.HDR, 'remote_flags')
+        else:
+            self.fielddescr_remote_flags = None
         frame_tid = self.layoutbuilder.get_type_id(jitframe.JITFRAME)
         self.translator._jit2gc['frame_tid'] = frame_tid
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -868,6 +868,11 @@
             # produce a SETFIELD to initialize the GC header
             self.emit_setfield(v_newgcobj, ConstInt(tid),
                                descr=self.gc_ll_descr.fielddescr_tid)
+        if self.gc_ll_descr.fielddescr_remote_flags is not None:
+            # produce a SETFIELD to initialize the rest of the GC header
+            # (Only present in _remoteheader GC)
+            self.emit_setfield(v_newgcobj, ConstInt(0),
+                               descr=self.gc_ll_descr.fielddescr_remote_flags)
 
     def gen_initialize_len(self, v_newgcobj, v_length, arraylen_descr):
         # produce a SETFIELD to initialize the array length
diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
@@ -84,6 +84,8 @@
     #
     t = TranslationContext()
     t.config.translation.gc = gc
+    # TODO: re-enable inlining when this test passes
+    t.config.translation.backendopt.inline = False
     if gc != 'boehm':
         t.config.translation.gcremovetypeptr = True
     for name, value in kwds.items():
diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -4,3 +4,8 @@
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
     gc = "incminimark"
+
+
+class TestRemoteHeaderShadowStack(CompileFrameworkTests):
+    gcrootfinder = "shadowstack"
+    gc = "incminimark_remoteheader"
diff --git a/rpython/jit/backend/x86/test/test_zrpy_vmprof.py b/rpython/jit/backend/x86/test/test_zrpy_vmprof.py
--- a/rpython/jit/backend/x86/test/test_zrpy_vmprof.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_vmprof.py
@@ -2,6 +2,9 @@
 from rpython.jit.backend.llsupport.test.zrpy_vmprof_test import CompiledVmprofTest
 
 class TestZVMprof(CompiledVmprofTest):
-    
     gcrootfinder = "shadowstack"
-    gc = "incminimark"
\ No newline at end of file
+    gc = "incminimark"
+
+class TestRemoteHeaderZVMprof(CompiledVmprofTest):
+    gcrootfinder = "shadowstack"
+    gc = "incminimark_remoteheader"
diff --git a/rpython/jit/backend/x86/test/test_zvmprof.py b/rpython/jit/backend/x86/test/test_zvmprof.py
--- a/rpython/jit/backend/x86/test/test_zvmprof.py
+++ b/rpython/jit/backend/x86/test/test_zvmprof.py
@@ -2,6 +2,9 @@
 from rpython.jit.backend.llsupport.test.zrpy_vmprof_test import CompiledVmprofTest
 
 class TestZVMprof(CompiledVmprofTest):
-    
     gcrootfinder = "shadowstack"
-    gc = "incminimark"
\ No newline at end of file
+    gc = "incminimark"
+
+class TestRemoteHeaderZVMprof(CompiledVmprofTest):
+    gcrootfinder = "shadowstack"
+    gc = "incminimark_remoteheader"
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -21,6 +21,7 @@
     can_usually_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    has_remote_flags = False
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -625,6 +625,7 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
+            ll_assert(self.nursery_free != llmemory.NULL, "malloc_fixedsize() with uninitialized nursery")
             result = self.nursery_free
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
@@ -685,6 +686,7 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
+            ll_assert(self.nursery_free != llmemory.NULL, "malloc_varsize() without initializing nursery")
             result = self.nursery_free
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
@@ -1088,6 +1090,7 @@
         # have been chosen to allow 'flags' to be zero in the common
         # case (hence the 'NO' in their name).
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        ll_assert(hdr != lltype.nullptr(self.HDR), "init_gc_object(): addr==NULL")
         hdr.tid = self.combine(typeid16, flags)
 
     def init_gc_object_immortal(self, addr, typeid16, flags=0):
@@ -1120,7 +1123,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj), "is_forwarded(): object not in nursery!")
+
         tid = self.get_flags(obj)
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1439,7 +1443,8 @@
                 self.add_flags(addr_array, GCFLAG_CARDS_SET)
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "_init_writebarrier_with_card_marker(): card_page_indices <= 0")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1523,7 +1528,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "_init_writebarrier_with_card_marker(): card_page_indices <= 0")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1684,12 +1690,13 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "_minor_collection(): self.pinned_objects_in_nursery != self.surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(cur >= prev, "_minor_collection(): cur < prev")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1747,7 +1754,7 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.get_flags(obj) & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.get_flags(obj) & GCFLAG_PINNED_OBJECT_PARENT_KNOWN, "_reset_flag_old_objects_pointing_to_pinned(): not GCFLAG_PINNED_OBJECT_PARENT_KNOWN")
         self.remove_flags(obj, GCFLAG_PINNED_OBJECT_PARENT_KNOWN)
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
@@ -3056,7 +3063,6 @@
     gc.finalize_header(hdr)
     return True
 
-
 class IncrementalMiniMarkGC(IncrementalMiniMarkGCBase):
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     # During a minor collection, the objects in the nursery that are
diff --git a/rpython/memory/gc/incminimark_remoteheader.py b/rpython/memory/gc/incminimark_remoteheader.py
--- a/rpython/memory/gc/incminimark_remoteheader.py
+++ b/rpython/memory/gc/incminimark_remoteheader.py
@@ -4,15 +4,19 @@
 from rpython.memory.gc import incminimark
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
+from rpython.rlib.debug import ll_assert
 
 SIGNEDP = lltype.Ptr(lltype.FixedSizeArray(lltype.Signed, 1))
 
 class IncrementalMiniMarkRemoteHeaderGC(incminimark.IncrementalMiniMarkGCBase):
     # The GC header is similar to incminimark, except that the flags can be
     # placed anywhere, not just in the bits of tid.
-    HDR = lltype.Struct('header',
-                        ('tid', lltype.Signed),
-                        ('remote_flags', SIGNEDP))
+    HDR = lltype.Struct(
+        'header',
+        ('tid', lltype.Signed),
+        ('remote_flags', SIGNEDP)
+    )
+    has_remote_flags = True
     minimal_size_in_nursery = (
         llmemory.sizeof(HDR) + llmemory.sizeof(llmemory.Address))
 
@@ -35,21 +39,22 @@
         # This gets compiled to nonsense like (&pypy_g_header_1433.h_tid)
         # at the top level (global variable initialization). Instead, we set
         # it to NULL and lazily initialize it later.
-        ## hdr.remote_flags = lltype.direct_fieldptr(hdr, 'tid')
         hdr.remote_flags = lltype.nullptr(SIGNEDP.TO)
 
     def make_forwardstub(self, obj, forward_to):
-        assert (self.header(obj).remote_flags
-                == lltype.direct_fieldptr(self.header(obj), 'tid')), \
-            "Nursery objects should not have separately-allocated flags."
+        hdr = self.header(obj)
+        ll_assert(
+            hdr.remote_flags == lltype.nullptr(SIGNEDP.TO)
+            or hdr.remote_flags == lltype.direct_fieldptr(hdr, 'tid'),
+            "Nursery objects should not have separately-allocated flags.")
         incminimark.IncrementalMiniMarkGCBase.make_forwardstub(self, obj, forward_to)
-        hdr = self.header(obj)
-        hdr.remote_flags = lltype.direct_fieldptr(hdr, 'tid')
 
     def copy_header(self, src, dest):
         dest_hdr = self.header(dest)
         dest_hdr.tid = self.get_flags(src)
-        dest_hdr.remote_flags = lltype.direct_fieldptr(dest_hdr, 'tid')
+        ll_assert(
+            not self.is_in_nursery(dest),
+            "Copying headers to another nursery element?")
         self.__extract_flags_to_pointer(dest_hdr)
 
     def __extract_flags_to_pointer(self, hdr):
@@ -57,9 +62,6 @@
 
         Expects the object to not use inline tid-flags.
         """
-        assert (hdr.remote_flags == lltype.nullptr(SIGNEDP.TO)
-                or hdr.remote_flags == lltype.direct_fieldptr(hdr, 'tid')), \
-                    "leaking old remote_flags!"
         size = llmemory.sizeof(lltype.Signed)
         adr = self.__ac_for_flags.malloc(size)
         hdr.remote_flags = llmemory.cast_adr_to_ptr(adr, SIGNEDP)
@@ -67,7 +69,8 @@
 
     def finalize_header(self, adr):
         hdr = llmemory.cast_adr_to_ptr(adr, lltype.Ptr(self.HDR))
-        if hdr.remote_flags != lltype.nullptr(SIGNEDP.TO):
+        if (hdr.remote_flags != lltype.nullptr(SIGNEDP.TO)
+            and hdr.remote_flags != lltype.direct_fieldptr(hdr, 'tid')):
             # If it points to allocated memory, this will be picked up by
             # __free_flags_if_finalized.
             hdr.remote_flags[0] |= incminimark.GCFLAG_DEAD
@@ -84,31 +87,28 @@
 
     # Manipulate flags through a pointer.
 
-    def __lazy_init_flags(self, hdr):
+    def __lazy_init_flags(self, obj):
+        hdr = self.header(obj)
         # XXX Is there anywhere I can initialize this only once without having
         #     to check for null on EVERY access?
         if hdr.remote_flags == lltype.nullptr(SIGNEDP.TO):
             hdr.remote_flags = lltype.direct_fieldptr(hdr, 'tid')
 
     def get_flags(self, obj):
-        hdr = self.header(obj)
-        self.__lazy_init_flags(hdr)
-        return hdr.remote_flags[0]
+        self.__lazy_init_flags(obj)
+        return self.header(obj).remote_flags[0]
 
     def set_flags(self, obj, flags):
-        hdr = self.header(obj)
-        self.__lazy_init_flags(hdr)
-        hdr.remote_flags[0] = flags
+        self.__lazy_init_flags(obj)
+        self.header(obj).remote_flags[0] = flags
 
     def add_flags(self, obj, flags):
-        hdr = self.header(obj)
-        self.__lazy_init_flags(hdr)
-        hdr.remote_flags[0] |= flags
+        self.__lazy_init_flags(obj)
+        self.header(obj).remote_flags[0] |= flags
 
     def remove_flags(self, obj, flags):
-        hdr = self.header(obj)
-        self.__lazy_init_flags(hdr)
-        hdr.remote_flags[0] &= ~flags
+        self.__lazy_init_flags(obj)
+        self.header(obj).remote_flags[0] &= ~flags
 
 
 def _free_flags_if_finalized(adr, unused_arg):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1660,7 +1660,7 @@
             assert res == 42
 
 
-class TestIncrementalMiniMarkRemoteHeadersGC(TestIncrementalMiniMarkGC):
+class TestIncrementalMiniMarkRemoteHeaderGC(TestIncrementalMiniMarkGC):
     gcpolicy = "incminimark_remoteheader"
 
 
@@ -1760,5 +1760,5 @@
 class TestIncrementalMiniMarkGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkGC):
     removetypeptr = True
 
-class TestIncrementalMiniMarkRemoteHeadersGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkRemoteHeadersGC):
+class TestIncrementalMiniMarkRemoteHeaderGCMostCompact(TaggedPointersTest, TestIncrementalMiniMarkRemoteHeaderGC):
     removetypeptr = True

From pypy.commits at gmail.com  Thu Jun  9 04:47:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Jun 2016 01:47:43 -0700 (PDT)
Subject: [pypy-commit] pypy incminimark-ll_assert: Close branch
 incminimark-ll_assert
Message-ID: <57592d2f.e1efc20a.3ad34.ffffff08@mx.google.com>

Author: Armin Rigo 
Branch: incminimark-ll_assert
Changeset: r85053:9c3395278354
Date: 2016-06-09 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/9c3395278354/

Log:	Close branch incminimark-ll_assert


From pypy.commits at gmail.com  Thu Jun  9 04:48:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Jun 2016 01:48:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in incminimark-ll_assert (pull
 request #453)
Message-ID: <57592d55.4275c20a.63b1e.06f9@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85054:d0457c3b2fed
Date: 2016-06-09 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/d0457c3b2fed/

Log:	Merged in incminimark-ll_assert (pull request #453)

	Use ll_assert (more often) in incminimark

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -281,11 +281,12 @@
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
+        "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
-        
+
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
@@ -644,6 +645,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -703,6 +705,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -1139,7 +1142,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj),
+                  "Can't forward an object outside the nursery.")
         tid = self.header(obj).tid
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1463,7 +1467,8 @@
                 objhdr.tid |= GCFLAG_CARDS_SET
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1513,7 +1518,8 @@
             return True
         # ^^^ a fast path of write-barrier
         #
-        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+        if (self.card_page_indices > 0 and     # check constant-folded
+            source_hdr.tid & GCFLAG_HAS_CARDS != 0):
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1548,7 +1554,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1721,12 +1728,15 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(
+            self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "pinned_objects_in_nursery != surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(
+                cur >= prev, "pinned objects encountered in backwards order")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1784,7 +1794,8 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
+                  "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):

From pypy.commits at gmail.com  Thu Jun  9 04:40:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Jun 2016 01:40:09 -0700 (PDT)
Subject: [pypy-commit] pypy incminimark-ll_assert: Small fixes,
 some of which are shown by failing tests before translation
Message-ID: <57592b69.04251c0a.b4768.ffff8dcb@mx.google.com>

Author: Armin Rigo 
Branch: incminimark-ll_assert
Changeset: r85052:94beb16f901f
Date: 2016-06-09 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/94beb16f901f/

Log:	Small fixes, some of which are shown by failing tests before
	translation

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -644,9 +644,8 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            ll_assert(self.nursery_free != llmemory.NULL,
-                      "uninitialized nursery")
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -705,9 +704,8 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            ll_assert(self.nursery_free != llmemory.NULL,
-                      "uninitialized nursery")
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -1520,7 +1518,8 @@
             return True
         # ^^^ a fast path of write-barrier
         #
-        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+        if (self.card_page_indices > 0 and     # check constant-folded
+            source_hdr.tid & GCFLAG_HAS_CARDS != 0):
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1795,7 +1794,7 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN,
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
                   "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 

From pypy.commits at gmail.com  Thu Jun  9 05:03:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Jun 2016 02:03:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test (at least on x86)
Message-ID: <575930fd.442cc20a.d70fb.0bae@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85055:541e5915479a
Date: 2016-06-09 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/541e5915479a/

Log:	Fix test (at least on x86)

diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -324,17 +324,19 @@
         def check(frame):
             expected_size = 1
             idx = 0
+            fixed_size = self.cpu.JITFRAME_FIXED_SIZE
             if self.cpu.backend_name.startswith('arm'):
                 # jitframe fixed part is larger here
                 expected_size = 2
                 idx = 1
+                fixed_size -= 32
             assert len(frame.jf_gcmap) == expected_size
-            if self.cpu.IS_64_BIT:
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE + 1  # +1 from i0
-            else:
-                assert frame.jf_gcmap[idx]
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE - 32 * idx + 1 # +1 from i0
-            assert frame.jf_gcmap[idx] == (1 << (exp_idx + 1)) | (1 << exp_idx)
+            # check that we have two bits set, and that they are in two
+            # registers (p0 and p1 are moved away when doing p2, but not
+            # spilled, just moved to different registers)
+            bits = [n for n in range(fixed_size)
+                      if frame.jf_gcmap[idx] & (1<

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85056:c6360ae83f41
Date: 2016-06-09 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/c6360ae83f41/

Log:	Remove starargs and kwargs from symtable

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -382,10 +382,6 @@
         self.note_symbol(clsdef.name, SYM_ASSIGNED)
         self.visit_sequence(clsdef.bases)
         self.visit_sequence(clsdef.keywords)
-        if clsdef.starargs:
-            clsdef.starargs.walkabout(self)
-        if clsdef.kwargs:
-            clsdef.kwargs.walkabout(self)
         self.visit_sequence(clsdef.decorator_list)
         self.push_scope(ClassScope(clsdef), clsdef)
         self.note_symbol('__class__', SYM_ASSIGNED)

From pypy.commits at gmail.com  Thu Jun  9 11:56:08 2016
From: pypy.commits at gmail.com (vext01)
Date: Thu, 09 Jun 2016 08:56:08 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-openbsd: Make vmprof build on OpenBSD.
Message-ID: <57599198.541a1c0a.c9402.fffff3a9@mx.google.com>

Author: Edd Barrett 
Branch: vmprof-openbsd
Changeset: r85057:b30d44ba76f7
Date: 2016-06-09 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/b30d44ba76f7/

Log:	Make vmprof build on OpenBSD.

diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,14 @@
+#if !defined(__OpenBSD__)
 #define HAVE_SYS_UCONTEXT_H
+#endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,8 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(__OpenBSD__)
+#include 
 #endif
 
 

From pypy.commits at gmail.com  Thu Jun  9 12:53:07 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 09 Jun 2016 09:53:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix asdl_py.py to work with new asdl.py,
 compile new ast.py
Message-ID: <57599ef3.06321c0a.d5f2.0ee8@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85058:f7c942bfac43
Date: 2016-06-09 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/f7c942bfac43/

Log:	Fix asdl_py.py to work with new asdl.py, compile new ast.py

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -16,7 +16,7 @@
     if not (space.isinstance_w(w_obj, space.w_str) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
-                    "AST string must be of type str or unicode")
+                   "AST string must be of type str or unicode")
     return w_obj
 
 def get_field(space, w_node, name, optional):
@@ -172,7 +172,7 @@
             return Suite.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected mod node, got %T", w_node)
-State.ast_type('mod', 'AST', None, [])
+State.ast_type(''mod'', 'AST', None, [])
 
 class Module(mod):
 
@@ -204,7 +204,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(_body)
 
-State.ast_type('Module', 'mod', ['body'])
+State.ast_type(''Module'', 'mod', ["'body'"])
 
 
 class Interactive(mod):
@@ -237,7 +237,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(_body)
 
-State.ast_type('Interactive', 'mod', ['body'])
+State.ast_type(''Interactive'', 'mod', ["'body'"])
 
 
 class Expression(mod):
@@ -266,7 +266,7 @@
             raise_required_value(space, w_node, 'body')
         return Expression(_body)
 
-State.ast_type('Expression', 'mod', ['body'])
+State.ast_type(''Expression'', 'mod', ["'body'"])
 
 
 class Suite(mod):
@@ -299,7 +299,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(_body)
 
-State.ast_type('Suite', 'mod', ['body'])
+State.ast_type(''Suite'', 'mod', ["'body'"])
 
 
 class stmt(AST):
@@ -356,7 +356,7 @@
             return Continue.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected stmt node, got %T", w_node)
-State.ast_type('stmt', 'AST', None, ['lineno', 'col_offset'])
+State.ast_type(''stmt'', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class FunctionDef(stmt):
 
@@ -431,17 +431,15 @@
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
 
-State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list', 'returns'])
+State.ast_type(''FunctionDef'', 'stmt', ["'name'", "'args'", "'body'", "'decorator_list'", "'returns'"])
 
 
 class ClassDef(stmt):
 
-    def __init__(self, name, bases, keywords, starargs, kwargs, body, decorator_list, lineno, col_offset):
+    def __init__(self, name, bases, keywords, body, decorator_list, lineno, col_offset):
         self.name = name
         self.bases = bases
         self.keywords = keywords
-        self.starargs = starargs
-        self.kwargs = kwargs
         self.body = body
         self.decorator_list = decorator_list
         stmt.__init__(self, lineno, col_offset)
@@ -454,10 +452,6 @@
             visitor._mutate_sequence(self.bases)
         if self.keywords:
             visitor._mutate_sequence(self.keywords)
-        if self.starargs:
-            self.starargs = self.starargs.mutate_over(visitor)
-        if self.kwargs:
-            self.kwargs = self.kwargs.mutate_over(visitor)
         if self.body:
             visitor._mutate_sequence(self.body)
         if self.decorator_list:
@@ -480,10 +474,6 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         if self.body is None:
             body_w = []
         else:
@@ -507,8 +497,6 @@
         w_name = get_field(space, w_node, 'name', False)
         w_bases = get_field(space, w_node, 'bases', False)
         w_keywords = get_field(space, w_node, 'keywords', False)
-        w_starargs = get_field(space, w_node, 'starargs', True)
-        w_kwargs = get_field(space, w_node, 'kwargs', True)
         w_body = get_field(space, w_node, 'body', False)
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
@@ -520,17 +508,15 @@
         _bases = [expr.from_object(space, w_item) for w_item in bases_w]
         keywords_w = space.unpackiterable(w_keywords)
         _keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
-        _starargs = expr.from_object(space, w_starargs) if w_starargs is not None else None
-        _kwargs = expr.from_object(space, w_kwargs) if w_kwargs is not None else None
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         decorator_list_w = space.unpackiterable(w_decorator_list)
         _decorator_list = [expr.from_object(space, w_item) for w_item in decorator_list_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return ClassDef(_name, _bases, _keywords, _starargs, _kwargs, _body, _decorator_list, _lineno, _col_offset)
-
-State.ast_type('ClassDef', 'stmt', ['name', 'bases', 'keywords', 'starargs', 'kwargs', 'body', 'decorator_list'])
+        return ClassDef(_name, _bases, _keywords, _body, _decorator_list, _lineno, _col_offset)
+
+State.ast_type(''ClassDef'', 'stmt', ["'name'", "'bases'", "'keywords'", "'body'", "'decorator_list'"])
 
 
 class Return(stmt):
@@ -567,7 +553,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
 
-State.ast_type('Return', 'stmt', ['value'])
+State.ast_type(''Return'', 'stmt', ["'value'"])
 
 
 class Delete(stmt):
@@ -609,7 +595,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Delete(_targets, _lineno, _col_offset)
 
-State.ast_type('Delete', 'stmt', ['targets'])
+State.ast_type(''Delete'', 'stmt', ["'targets'"])
 
 
 class Assign(stmt):
@@ -659,7 +645,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assign(_targets, _value, _lineno, _col_offset)
 
-State.ast_type('Assign', 'stmt', ['targets', 'value'])
+State.ast_type(''Assign'', 'stmt', ["'targets'", "'value'"])
 
 
 class AugAssign(stmt):
@@ -712,7 +698,7 @@
         _col_offset = space.int_w(w_col_offset)
         return AugAssign(_target, _op, _value, _lineno, _col_offset)
 
-State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value'])
+State.ast_type(''AugAssign'', 'stmt', ["'target'", "'op'", "'value'"])
 
 
 class For(stmt):
@@ -782,7 +768,7 @@
         _col_offset = space.int_w(w_col_offset)
         return For(_target, _iter, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse'])
+State.ast_type(''For'', 'stmt', ["'target'", "'iter'", "'body'", "'orelse'"])
 
 
 class While(stmt):
@@ -844,7 +830,7 @@
         _col_offset = space.int_w(w_col_offset)
         return While(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('While', 'stmt', ['test', 'body', 'orelse'])
+State.ast_type(''While'', 'stmt', ["'test'", "'body'", "'orelse'"])
 
 
 class If(stmt):
@@ -906,7 +892,7 @@
         _col_offset = space.int_w(w_col_offset)
         return If(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('If', 'stmt', ['test', 'body', 'orelse'])
+State.ast_type(''If'', 'stmt', ["'test'", "'body'", "'orelse'"])
 
 
 class With(stmt):
@@ -960,7 +946,7 @@
         _col_offset = space.int_w(w_col_offset)
         return With(_items, _body, _lineno, _col_offset)
 
-State.ast_type('With', 'stmt', ['items', 'body'])
+State.ast_type(''With'', 'stmt', ["'items'", "'body'"])
 
 
 class Raise(stmt):
@@ -1004,7 +990,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
 
-State.ast_type('Raise', 'stmt', ['exc', 'cause'])
+State.ast_type(''Raise'', 'stmt', ["'exc'", "'cause'"])
 
 
 class Try(stmt):
@@ -1082,7 +1068,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Try(_body, _handlers, _orelse, _finalbody, _lineno, _col_offset)
 
-State.ast_type('Try', 'stmt', ['body', 'handlers', 'orelse', 'finalbody'])
+State.ast_type(''Try'', 'stmt', ["'body'", "'handlers'", "'orelse'", "'finalbody'"])
 
 
 class Assert(stmt):
@@ -1127,7 +1113,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
 
-State.ast_type('Assert', 'stmt', ['test', 'msg'])
+State.ast_type(''Assert'', 'stmt', ["'test'", "'msg'"])
 
 
 class Import(stmt):
@@ -1169,7 +1155,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Import(_names, _lineno, _col_offset)
 
-State.ast_type('Import', 'stmt', ['names'])
+State.ast_type(''Import'', 'stmt', ["'names'"])
 
 
 class ImportFrom(stmt):
@@ -1221,7 +1207,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ImportFrom(_module, _names, _level, _lineno, _col_offset)
 
-State.ast_type('ImportFrom', 'stmt', ['module', 'names', 'level'])
+State.ast_type(''ImportFrom'', 'stmt', ["'module'", "'names'", "'level'"])
 
 
 class Global(stmt):
@@ -1261,7 +1247,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
 
-State.ast_type('Global', 'stmt', ['names'])
+State.ast_type(''Global'', 'stmt', ["'names'"])
 
 
 class Nonlocal(stmt):
@@ -1301,7 +1287,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Nonlocal(_names, _lineno, _col_offset)
 
-State.ast_type('Nonlocal', 'stmt', ['names'])
+State.ast_type(''Nonlocal'', 'stmt', ["'names'"])
 
 
 class Expr(stmt):
@@ -1339,7 +1325,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Expr(_value, _lineno, _col_offset)
 
-State.ast_type('Expr', 'stmt', ['value'])
+State.ast_type(''Expr'', 'stmt', ["'value'"])
 
 
 class Pass(stmt):
@@ -1369,7 +1355,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Pass(_lineno, _col_offset)
 
-State.ast_type('Pass', 'stmt', [])
+State.ast_type(''Pass'', 'stmt', [])
 
 
 class Break(stmt):
@@ -1399,7 +1385,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Break(_lineno, _col_offset)
 
-State.ast_type('Break', 'stmt', [])
+State.ast_type(''Break'', 'stmt', [])
 
 
 class Continue(stmt):
@@ -1429,7 +1415,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Continue(_lineno, _col_offset)
 
-State.ast_type('Continue', 'stmt', [])
+State.ast_type(''Continue'', 'stmt', [])
 
 
 class expr(AST):
@@ -1478,6 +1464,8 @@
             return Str.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Bytes):
             return Bytes.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_NameConstant):
+            return NameConstant.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Ellipsis):
             return Ellipsis.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Attribute):
@@ -1496,7 +1484,7 @@
             return Const.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected expr node, got %T", w_node)
-State.ast_type('expr', 'AST', None, ['lineno', 'col_offset'])
+State.ast_type(''expr'', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class BoolOp(expr):
 
@@ -1544,7 +1532,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BoolOp(_op, _values, _lineno, _col_offset)
 
-State.ast_type('BoolOp', 'expr', ['op', 'values'])
+State.ast_type(''BoolOp'', 'expr', ["'op'", "'values'"])
 
 
 class BinOp(expr):
@@ -1597,7 +1585,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BinOp(_left, _op, _right, _lineno, _col_offset)
 
-State.ast_type('BinOp', 'expr', ['left', 'op', 'right'])
+State.ast_type(''BinOp'', 'expr', ["'left'", "'op'", "'right'"])
 
 
 class UnaryOp(expr):
@@ -1642,7 +1630,7 @@
         _col_offset = space.int_w(w_col_offset)
         return UnaryOp(_op, _operand, _lineno, _col_offset)
 
-State.ast_type('UnaryOp', 'expr', ['op', 'operand'])
+State.ast_type(''UnaryOp'', 'expr', ["'op'", "'operand'"])
 
 
 class Lambda(expr):
@@ -1688,7 +1676,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Lambda(_args, _body, _lineno, _col_offset)
 
-State.ast_type('Lambda', 'expr', ['args', 'body'])
+State.ast_type(''Lambda'', 'expr', ["'args'", "'body'"])
 
 
 class IfExp(expr):
@@ -1742,7 +1730,7 @@
         _col_offset = space.int_w(w_col_offset)
         return IfExp(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('IfExp', 'expr', ['test', 'body', 'orelse'])
+State.ast_type(''IfExp'', 'expr', ["'test'", "'body'", "'orelse'"])
 
 
 class Dict(expr):
@@ -1796,7 +1784,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Dict(_keys, _values, _lineno, _col_offset)
 
-State.ast_type('Dict', 'expr', ['keys', 'values'])
+State.ast_type(''Dict'', 'expr', ["'keys'", "'values'"])
 
 
 class Set(expr):
@@ -1838,7 +1826,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Set(_elts, _lineno, _col_offset)
 
-State.ast_type('Set', 'expr', ['elts'])
+State.ast_type(''Set'', 'expr', ["'elts'"])
 
 
 class ListComp(expr):
@@ -1888,7 +1876,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ListComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('ListComp', 'expr', ['elt', 'generators'])
+State.ast_type(''ListComp'', 'expr', ["'elt'", "'generators'"])
 
 
 class SetComp(expr):
@@ -1938,7 +1926,7 @@
         _col_offset = space.int_w(w_col_offset)
         return SetComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('SetComp', 'expr', ['elt', 'generators'])
+State.ast_type(''SetComp'', 'expr', ["'elt'", "'generators'"])
 
 
 class DictComp(expr):
@@ -1996,7 +1984,7 @@
         _col_offset = space.int_w(w_col_offset)
         return DictComp(_key, _value, _generators, _lineno, _col_offset)
 
-State.ast_type('DictComp', 'expr', ['key', 'value', 'generators'])
+State.ast_type(''DictComp'', 'expr', ["'key'", "'value'", "'generators'"])
 
 
 class GeneratorExp(expr):
@@ -2046,7 +2034,7 @@
         _col_offset = space.int_w(w_col_offset)
         return GeneratorExp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('GeneratorExp', 'expr', ['elt', 'generators'])
+State.ast_type(''GeneratorExp'', 'expr', ["'elt'", "'generators'"])
 
 
 class Yield(expr):
@@ -2083,7 +2071,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
 
-State.ast_type('Yield', 'expr', ['value'])
+State.ast_type(''Yield'', 'expr', ["'value'"])
 
 
 class YieldFrom(expr):
@@ -2121,7 +2109,7 @@
         _col_offset = space.int_w(w_col_offset)
         return YieldFrom(_value, _lineno, _col_offset)
 
-State.ast_type('YieldFrom', 'expr', ['value'])
+State.ast_type(''YieldFrom'', 'expr', ["'value'"])
 
 
 class Compare(expr):
@@ -2181,17 +2169,15 @@
         _col_offset = space.int_w(w_col_offset)
         return Compare(_left, _ops, _comparators, _lineno, _col_offset)
 
-State.ast_type('Compare', 'expr', ['left', 'ops', 'comparators'])
+State.ast_type(''Compare'', 'expr', ["'left'", "'ops'", "'comparators'"])
 
 
 class Call(expr):
 
-    def __init__(self, func, args, keywords, starargs, kwargs, lineno, col_offset):
+    def __init__(self, func, args, keywords, lineno, col_offset):
         self.func = func
         self.args = args
         self.keywords = keywords
-        self.starargs = starargs
-        self.kwargs = kwargs
         expr.__init__(self, lineno, col_offset)
 
     def walkabout(self, visitor):
@@ -2203,10 +2189,6 @@
             visitor._mutate_sequence(self.args)
         if self.keywords:
             visitor._mutate_sequence(self.keywords)
-        if self.starargs:
-            self.starargs = self.starargs.mutate_over(visitor)
-        if self.kwargs:
-            self.kwargs = self.kwargs.mutate_over(visitor)
         return visitor.visit_Call(self)
 
     def to_object(self, space):
@@ -2225,10 +2207,6 @@
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
         space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('starargs'), w_starargs)
-        w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
         w_col_offset = space.wrap(self.col_offset)  # int
@@ -2240,8 +2218,6 @@
         w_func = get_field(space, w_node, 'func', False)
         w_args = get_field(space, w_node, 'args', False)
         w_keywords = get_field(space, w_node, 'keywords', False)
-        w_starargs = get_field(space, w_node, 'starargs', True)
-        w_kwargs = get_field(space, w_node, 'kwargs', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _func = expr.from_object(space, w_func)
@@ -2251,13 +2227,11 @@
         _args = [expr.from_object(space, w_item) for w_item in args_w]
         keywords_w = space.unpackiterable(w_keywords)
         _keywords = [keyword.from_object(space, w_item) for w_item in keywords_w]
-        _starargs = expr.from_object(space, w_starargs) if w_starargs is not None else None
-        _kwargs = expr.from_object(space, w_kwargs) if w_kwargs is not None else None
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
-        return Call(_func, _args, _keywords, _starargs, _kwargs, _lineno, _col_offset)
-
-State.ast_type('Call', 'expr', ['func', 'args', 'keywords', 'starargs', 'kwargs'])
+        return Call(_func, _args, _keywords, _lineno, _col_offset)
+
+State.ast_type(''Call'', 'expr', ["'func'", "'args'", "'keywords'"])
 
 
 class Num(expr):
@@ -2294,7 +2268,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Num(_n, _lineno, _col_offset)
 
-State.ast_type('Num', 'expr', ['n'])
+State.ast_type(''Num'', 'expr', ["'n'"])
 
 
 class Str(expr):
@@ -2331,7 +2305,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Str(_s, _lineno, _col_offset)
 
-State.ast_type('Str', 'expr', ['s'])
+State.ast_type(''Str'', 'expr', ["'s'"])
 
 
 class Bytes(expr):
@@ -2348,7 +2322,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Bytes)
-        w_s = self.s  # string
+        w_s = self.s.to_object(space)  # bytes
         space.setattr(w_node, space.wrap('s'), w_s)
         w_lineno = space.wrap(self.lineno)  # int
         space.setattr(w_node, space.wrap('lineno'), w_lineno)
@@ -2361,14 +2335,51 @@
         w_s = get_field(space, w_node, 's', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _s = check_string(space, w_s)
+        _s = bytes.from_object(space, w_s)
         if _s is None:
             raise_required_value(space, w_node, 's')
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Bytes(_s, _lineno, _col_offset)
 
-State.ast_type('Bytes', 'expr', ['s'])
+State.ast_type(''Bytes'', 'expr', ["'s'"])
+
+
+class NameConstant(expr):
+
+    def __init__(self, value, lineno, col_offset):
+        self.value = value
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_NameConstant(self)
+
+    def mutate_over(self, visitor):
+        return visitor.visit_NameConstant(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_NameConstant)
+        w_value = self.value.to_object(space)  # singleton
+        space.setattr(w_node, space.wrap('value'), w_value)
+        w_lineno = space.wrap(self.lineno)  # int
+        space.setattr(w_node, space.wrap('lineno'), w_lineno)
+        w_col_offset = space.wrap(self.col_offset)  # int
+        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_value = get_field(space, w_node, 'value', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _value = singleton.from_object(space, w_value)
+        if _value is None:
+            raise_required_value(space, w_node, 'value')
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return NameConstant(_value, _lineno, _col_offset)
+
+State.ast_type(''NameConstant'', 'expr', ["'value'"])
 
 
 class Ellipsis(expr):
@@ -2398,7 +2409,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Ellipsis(_lineno, _col_offset)
 
-State.ast_type('Ellipsis', 'expr', [])
+State.ast_type(''Ellipsis'', 'expr', [])
 
 
 class Attribute(expr):
@@ -2450,7 +2461,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
 
-State.ast_type('Attribute', 'expr', ['value', 'attr', 'ctx'])
+State.ast_type(''Attribute'', 'expr', ["'value'", "'attr'", "'ctx'"])
 
 
 class Subscript(expr):
@@ -2503,7 +2514,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
 
-State.ast_type('Subscript', 'expr', ['value', 'slice', 'ctx'])
+State.ast_type(''Subscript'', 'expr', ["'value'", "'slice'", "'ctx'"])
 
 
 class Starred(expr):
@@ -2548,7 +2559,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Starred(_value, _ctx, _lineno, _col_offset)
 
-State.ast_type('Starred', 'expr', ['value', 'ctx'])
+State.ast_type(''Starred'', 'expr', ["'value'", "'ctx'"])
 
 
 class Name(expr):
@@ -2592,7 +2603,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Name(_id, _ctx, _lineno, _col_offset)
 
-State.ast_type('Name', 'expr', ['id', 'ctx'])
+State.ast_type(''Name'', 'expr', ["'id'", "'ctx'"])
 
 
 class List(expr):
@@ -2641,7 +2652,7 @@
         _col_offset = space.int_w(w_col_offset)
         return List(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type('List', 'expr', ['elts', 'ctx'])
+State.ast_type(''List'', 'expr', ["'elts'", "'ctx'"])
 
 
 class Tuple(expr):
@@ -2690,7 +2701,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Tuple(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type('Tuple', 'expr', ['elts', 'ctx'])
+State.ast_type(''Tuple'', 'expr', ["'elts'", "'ctx'"])
 
 
 class Const(expr):
@@ -2727,7 +2738,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Const(_value, _lineno, _col_offset)
 
-State.ast_type('Const', 'expr', ['value'])
+State.ast_type(''Const'', 'expr', ["'value'"])
 
 
 class expr_context(AST):
@@ -2808,7 +2819,7 @@
             return Index.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected slice node, got %T", w_node)
-State.ast_type('slice', 'AST', None, [])
+State.ast_type(''slice'', 'AST', None, [])
 
 class Slice(slice):
 
@@ -2849,7 +2860,7 @@
         _step = expr.from_object(space, w_step) if w_step is not None else None
         return Slice(_lower, _upper, _step)
 
-State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
+State.ast_type(''Slice'', 'slice', ["'lower'", "'upper'", "'step'"])
 
 
 class ExtSlice(slice):
@@ -2882,7 +2893,7 @@
         _dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(_dims)
 
-State.ast_type('ExtSlice', 'slice', ['dims'])
+State.ast_type(''ExtSlice'', 'slice', ["'dims'"])
 
 
 class Index(slice):
@@ -2911,7 +2922,7 @@
             raise_required_value(space, w_node, 'value')
         return Index(_value)
 
-State.ast_type('Index', 'slice', ['value'])
+State.ast_type(''Index'', 'slice', ["'value'"])
 
 
 class boolop(AST):
@@ -2952,25 +2963,25 @@
             return 2
         if space.isinstance_w(w_node, get(space).w_Mult):
             return 3
+        if space.isinstance_w(w_node, get(space).w_MatMult):
+            return 4
         if space.isinstance_w(w_node, get(space).w_Div):
-            return 4
+            return 5
         if space.isinstance_w(w_node, get(space).w_Mod):
-            return 5
+            return 6
         if space.isinstance_w(w_node, get(space).w_Pow):
-            return 6
+            return 7
         if space.isinstance_w(w_node, get(space).w_LShift):
-            return 7
+            return 8
         if space.isinstance_w(w_node, get(space).w_RShift):
-            return 8
+            return 9
         if space.isinstance_w(w_node, get(space).w_BitOr):
-            return 9
+            return 10
         if space.isinstance_w(w_node, get(space).w_BitXor):
-            return 10
+            return 11
         if space.isinstance_w(w_node, get(space).w_BitAnd):
-            return 11
+            return 12
         if space.isinstance_w(w_node, get(space).w_FloorDiv):
-            return 12
-        if space.isinstance_w(w_node, get(space).w_MatMul):
             return 13
         raise oefmt(space.w_TypeError,
                 "Expected operator node, got %T", w_node)
@@ -2991,6 +3002,11 @@
         return space.call_function(get(space).w_Mult)
 State.ast_type('Mult', 'operator', None)
 
+class _MatMult(operator):
+    def to_object(self, space):
+        return space.call_function(get(space).w_MatMult)
+State.ast_type('MatMult', 'operator', None)
+
 class _Div(operator):
     def to_object(self, space):
         return space.call_function(get(space).w_Div)
@@ -3036,29 +3052,25 @@
         return space.call_function(get(space).w_FloorDiv)
 State.ast_type('FloorDiv', 'operator', None)
 
-class _MatMul(operator):
-    def to_object(self, space):
-        return space.call_function(get(space).w_MatMul)
-State.ast_type('MatMul', 'operator', None)
-
 Add = 1
 Sub = 2
 Mult = 3
-Div = 4
-Mod = 5
-Pow = 6
-LShift = 7
-RShift = 8
-BitOr = 9
-BitXor = 10
-BitAnd = 11
-FloorDiv = 12
-MatMul = 13
+MatMult = 4
+Div = 5
+Mod = 6
+Pow = 7
+LShift = 8
+RShift = 9
+BitOr = 10
+BitXor = 11
+BitAnd = 12
+FloorDiv = 13
 
 operator_to_class = [
     _Add,
     _Sub,
     _Mult,
+    _MatMult,
     _Div,
     _Mod,
     _Pow,
@@ -3068,7 +3080,6 @@
     _BitXor,
     _BitAnd,
     _FloorDiv,
-    _MatMul,
 ]
 
 class unaryop(AST):
@@ -3265,7 +3276,7 @@
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
 
-State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs'])
+State.ast_type(''comprehension'', 'AST', ["'target'", "'iter'", "'ifs'"])
 
 class excepthandler(AST):
 
@@ -3281,7 +3292,7 @@
             return ExceptHandler.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected excepthandler node, got %T", w_node)
-State.ast_type('excepthandler', 'AST', None, ['lineno', 'col_offset'])
+State.ast_type(''excepthandler'', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class ExceptHandler(excepthandler):
 
@@ -3334,34 +3345,32 @@
         _col_offset = space.int_w(w_col_offset)
         return ExceptHandler(_type, _name, _body, _lineno, _col_offset)
 
-State.ast_type('ExceptHandler', 'excepthandler', ['type', 'name', 'body'])
+State.ast_type(''ExceptHandler'', 'excepthandler', ["'type'", "'name'", "'body'"])
 
 
 class arguments(AST):
 
-    def __init__(self, args, vararg, varargannotation, kwonlyargs, kwarg, kwargannotation, defaults, kw_defaults):
+    def __init__(self, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults):
         self.args = args
         self.vararg = vararg
-        self.varargannotation = varargannotation
         self.kwonlyargs = kwonlyargs
+        self.kw_defaults = kw_defaults
         self.kwarg = kwarg
-        self.kwargannotation = kwargannotation
         self.defaults = defaults
-        self.kw_defaults = kw_defaults
 
     def mutate_over(self, visitor):
         if self.args:
             visitor._mutate_sequence(self.args)
-        if self.varargannotation:
-            self.varargannotation = self.varargannotation.mutate_over(visitor)
+        if self.vararg:
+            self.vararg = self.vararg.mutate_over(visitor)
         if self.kwonlyargs:
             visitor._mutate_sequence(self.kwonlyargs)
-        if self.kwargannotation:
-            self.kwargannotation = self.kwargannotation.mutate_over(visitor)
+        if self.kw_defaults:
+            visitor._mutate_sequence(self.kw_defaults)
+        if self.kwarg:
+            self.kwarg = self.kwarg.mutate_over(visitor)
         if self.defaults:
             visitor._mutate_sequence(self.defaults)
-        if self.kw_defaults:
-            visitor._mutate_sequence(self.kw_defaults)
         return visitor.visit_arguments(self)
 
     def walkabout(self, visitor):
@@ -3375,59 +3384,51 @@
             args_w = [node.to_object(space) for node in self.args] # arg
         w_args = space.newlist(args_w)
         space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = space.wrap(self.vararg.decode('utf-8')) if self.vararg is not None else space.w_None  # identifier
+        w_vararg = self.vararg.to_object(space) if self.vararg is not None else space.w_None  # arg
         space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_varargannotation = self.varargannotation.to_object(space) if self.varargannotation is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('varargannotation'), w_varargannotation)
         if self.kwonlyargs is None:
             kwonlyargs_w = []
         else:
             kwonlyargs_w = [node.to_object(space) for node in self.kwonlyargs] # arg
         w_kwonlyargs = space.newlist(kwonlyargs_w)
         space.setattr(w_node, space.wrap('kwonlyargs'), w_kwonlyargs)
-        w_kwarg = space.wrap(self.kwarg.decode('utf-8')) if self.kwarg is not None else space.w_None  # identifier
+        if self.kw_defaults is None:
+            kw_defaults_w = []
+        else:
+            kw_defaults_w = [node.to_object(space) if node is not None else space.w_None for node in self.kw_defaults] # expr
+        w_kw_defaults = space.newlist(kw_defaults_w)
+        space.setattr(w_node, space.wrap('kw_defaults'), w_kw_defaults)
+        w_kwarg = self.kwarg.to_object(space) if self.kwarg is not None else space.w_None  # arg
         space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
-        w_kwargannotation = self.kwargannotation.to_object(space) if self.kwargannotation is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargannotation'), w_kwargannotation)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
         space.setattr(w_node, space.wrap('defaults'), w_defaults)
-        if self.kw_defaults is None:
-            kw_defaults_w = []
-        else:
-            kw_defaults_w = [node.to_object(space) if node is not None else space.w_None for node in self.kw_defaults] # expr
-        w_kw_defaults = space.newlist(kw_defaults_w)
-        space.setattr(w_node, space.wrap('kw_defaults'), w_kw_defaults)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_args = get_field(space, w_node, 'args', False)
         w_vararg = get_field(space, w_node, 'vararg', True)
-        w_varargannotation = get_field(space, w_node, 'varargannotation', True)
         w_kwonlyargs = get_field(space, w_node, 'kwonlyargs', False)
+        w_kw_defaults = get_field(space, w_node, 'kw_defaults', False)
         w_kwarg = get_field(space, w_node, 'kwarg', True)
-        w_kwargannotation = get_field(space, w_node, 'kwargannotation', True)
         w_defaults = get_field(space, w_node, 'defaults', False)
-        w_kw_defaults = get_field(space, w_node, 'kw_defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [arg.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.str_or_None_w(w_vararg)
-        _varargannotation = expr.from_object(space, w_varargannotation) if w_varargannotation is not None else None
+        _vararg = arg.from_object(space, w_vararg) if w_vararg is not None else None
         kwonlyargs_w = space.unpackiterable(w_kwonlyargs)
         _kwonlyargs = [arg.from_object(space, w_item) for w_item in kwonlyargs_w]
-        _kwarg = space.str_or_None_w(w_kwarg)
-        _kwargannotation = expr.from_object(space, w_kwargannotation) if w_kwargannotation is not None else None
+        kw_defaults_w = space.unpackiterable(w_kw_defaults)
+        _kw_defaults = [expr.from_object(space, w_item) for w_item in kw_defaults_w]
+        _kwarg = arg.from_object(space, w_kwarg) if w_kwarg is not None else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
-        kw_defaults_w = space.unpackiterable(w_kw_defaults)
-        _kw_defaults = [expr.from_object(space, w_item) for w_item in kw_defaults_w]
-        return arguments(_args, _vararg, _varargannotation, _kwonlyargs, _kwarg, _kwargannotation, _defaults, _kw_defaults)
-
-State.ast_type('arguments', 'AST', ['args', 'vararg', 'varargannotation', 'kwonlyargs', 'kwarg', 'kwargannotation', 'defaults', 'kw_defaults'])
+        return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
+
+State.ast_type(''arguments'', 'AST', ["'args'", "'vararg'", "'kwonlyargs'", "'kw_defaults'", "'kwarg'", "'defaults'"])
 
 class arg(AST):
 
@@ -3461,7 +3462,7 @@
         _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
         return arg(_arg, _annotation)
 
-State.ast_type('arg', 'AST', ['arg', 'annotation'])
+State.ast_type(''arg'', 'AST', ["'arg'", "'annotation'"])
 
 class keyword(AST):
 
@@ -3478,7 +3479,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg.decode('utf-8'))  # identifier
+        w_arg = space.wrap(self.arg.decode('utf-8')) if self.arg is not None else space.w_None  # identifier
         space.setattr(w_node, space.wrap('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.wrap('value'), w_value)
@@ -3486,17 +3487,15 @@
 
     @staticmethod
     def from_object(space, w_node):
-        w_arg = get_field(space, w_node, 'arg', False)
+        w_arg = get_field(space, w_node, 'arg', True)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.identifier_w(w_arg)
-        if _arg is None:
-            raise_required_value(space, w_node, 'arg')
+        _arg = space.str_or_None_w(w_arg)
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
         return keyword(_arg, _value)
 
-State.ast_type('keyword', 'AST', ['arg', 'value'])
+State.ast_type(''keyword'', 'AST', ["'arg'", "'value'"])
 
 class alias(AST):
 
@@ -3528,7 +3527,7 @@
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
-State.ast_type('alias', 'AST', ['name', 'asname'])
+State.ast_type(''alias'', 'AST', ["'name'", "'asname'"])
 
 class withitem(AST):
 
@@ -3563,7 +3562,7 @@
         _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
         return withitem(_context_expr, _optional_vars)
 
-State.ast_type('withitem', 'AST', ['context_expr', 'optional_vars'])
+State.ast_type(''withitem'', 'AST', ["'context_expr'", "'optional_vars'"])
 
 class ASTVisitor(object):
 
@@ -3672,6 +3671,8 @@
         return self.default_visitor(node)
     def visit_Bytes(self, node):
         return self.default_visitor(node)
+    def visit_NameConstant(self, node):
+        return self.default_visitor(node)
     def visit_Ellipsis(self, node):
         return self.default_visitor(node)
     def visit_Attribute(self, node):
@@ -3733,10 +3734,6 @@
     def visit_ClassDef(self, node):
         self.visit_sequence(node.bases)
         self.visit_sequence(node.keywords)
-        if node.starargs:
-            node.starargs.walkabout(self)
-        if node.kwargs:
-            node.kwargs.walkabout(self)
         self.visit_sequence(node.body)
         self.visit_sequence(node.decorator_list)
 
@@ -3874,10 +3871,6 @@
         node.func.walkabout(self)
         self.visit_sequence(node.args)
         self.visit_sequence(node.keywords)
-        if node.starargs:
-            node.starargs.walkabout(self)
-        if node.kwargs:
-            node.kwargs.walkabout(self)
 
     def visit_Num(self, node):
         pass
@@ -3888,6 +3881,9 @@
     def visit_Bytes(self, node):
         pass
 
+    def visit_NameConstant(self, node):
+        pass
+
     def visit_Ellipsis(self, node):
         pass
 
@@ -3939,13 +3935,13 @@
 
     def visit_arguments(self, node):
         self.visit_sequence(node.args)
-        if node.varargannotation:
-            node.varargannotation.walkabout(self)
+        if node.vararg:
+            node.vararg.walkabout(self)
         self.visit_sequence(node.kwonlyargs)
-        if node.kwargannotation:
-            node.kwargannotation.walkabout(self)
+        self.visit_sequence(node.kw_defaults)
+        if node.kwarg:
+            node.kwarg.walkabout(self)
         self.visit_sequence(node.defaults)
-        self.visit_sequence(node.kw_defaults)
 
     def visit_arg(self, node):
         if node.annotation:
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -85,7 +85,7 @@
             self.emit("class %s(AST):" % (base,))
             if sum.attributes:
                 self.emit("")
-                args = ", ".join(attr.name.value for attr in sum.attributes)
+                args = ", ".join(attr.name for attr in sum.attributes)
                 self.emit("def __init__(self, %s):" % (args,), 1)
                 for attr in sum.attributes:
                     self.visit(attr)
@@ -123,39 +123,39 @@
         self.emit("")
 
     def get_value_converter(self, field, value):
-        if field.type.value in self.data.simple_types:
+        if field.type in self.data.simple_types:
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
-        elif field.type.value == "identifier":
+        elif field.type == "identifier":
             wrapper = "space.wrap(%s.decode('utf-8'))" % (value,)
             if field.opt:
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
-        elif field.type.value in ("object", "string"):
+        elif field.type in ("object", "string"):
             return value
-        elif field.type.value in ("int", "bool"):
+        elif field.type in ("int", "bool"):
             return "space.wrap(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             # XXX: kw_defaults, unlike other sequences, allows None
             # values
-            if field.opt or field.name.value == 'kw_defaults':
+            if field.opt or field.name == 'kw_defaults':
                 wrapper += " if %s is not None else space.w_None" % (value,)
             return wrapper
         
     def get_value_extractor(self, field, value):
-        if field.type.value in self.data.simple_types:
+        if field.type in self.data.simple_types:
             return "%s.from_object(space, %s)" % (field.type, value)
-        elif field.type.value in ("object",):
+        elif field.type in ("object",):
             return value
-        elif field.type.value in ("string",):
+        elif field.type in ("string",):
             return "check_string(space, %s)" % (value,)
-        elif field.type.value in ("identifier",):
+        elif field.type in ("identifier",):
             if field.opt:
                 return "space.str_or_None_w(%s)" % (value,)
             return "space.identifier_w(%s)" % (value,)
-        elif field.type.value in ("int",):
+        elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
-        elif field.type.value in ("bool",):
+        elif field.type in ("bool",):
             return "space.bool_w(%s)" % (value,)
         else:
             extractor = "%s.from_object(space, %s)" % (field.type, value)
@@ -189,7 +189,7 @@
         else:
             value = self.get_value_extractor(field, "w_%s" % (field.name,))
             lines = ["_%s = %s" % (field.name, value)]
-            if not field.opt and field.type.value not in ("int",):
+            if not field.opt and field.type not in ("int",):
                 lines.append("if _%s is None:" % (field.name,))
                 lines.append("    raise_required_value(space, w_node, '%s')"
                              % (field.name,))
@@ -235,8 +235,8 @@
     def make_mutate_over(self, cons, name):
         self.emit("def mutate_over(self, visitor):", 1)
         for field in cons.fields:
-            if (field.type.value not in asdl.builtin_types and
-                field.type.value not in self.data.simple_types):
+            if (field.type not in asdl.builtin_types and
+                field.type not in self.data.simple_types):
                 if field.opt or field.seq:
                     level = 3
                     self.emit("if self.%s:" % (field.name,), 2)
@@ -342,8 +342,8 @@
         self.emit("")
 
     def visitField(self, field):
-        if (field.type.value not in asdl.builtin_types and 
-            field.type.value not in self.data.simple_types):
+        if (field.type not in asdl.builtin_types and 
+            field.type not in self.data.simple_types):
             level = 2
             template = "node.%s.walkabout(self)"
             if field.seq:
@@ -384,7 +384,7 @@
             if isinstance(tp.value, asdl.Sum):
                 sum = tp.value
                 if is_simple_sum(sum):
-                    simple_types.add(tp.name.value)
+                    simple_types.add(tp.name)
                 else:
                     attrs = [field for field in sum.attributes]
                     for cons in sum.types:
@@ -392,7 +392,7 @@
                         cons_attributes[cons] = attrs
             else:
                 prod = tp.value
-                prod_simple.add(tp.name.value)
+                prod_simple.add(tp.name)
                 add_masks(prod.fields, prod)
         prod_simple.update(simple_types)
         self.cons_attributes = cons_attributes

From pypy.commits at gmail.com  Thu Jun  9 13:23:22 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 09 Jun 2016 10:23:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix formatting error because of new
 asdl.py
Message-ID: <5759a60a.832c1c0a.20d42.087b@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85059:af060ac191a1
Date: 2016-06-09 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/af060ac191a1/

Log:	Fix formatting error because of new asdl.py

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -172,7 +172,7 @@
             return Suite.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected mod node, got %T", w_node)
-State.ast_type(''mod'', 'AST', None, [])
+State.ast_type('mod', 'AST', None, [])
 
 class Module(mod):
 
@@ -204,7 +204,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(_body)
 
-State.ast_type(''Module'', 'mod', ["'body'"])
+State.ast_type('Module', 'mod', ["'body'"])
 
 
 class Interactive(mod):
@@ -237,7 +237,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(_body)
 
-State.ast_type(''Interactive'', 'mod', ["'body'"])
+State.ast_type('Interactive', 'mod', ["'body'"])
 
 
 class Expression(mod):
@@ -266,7 +266,7 @@
             raise_required_value(space, w_node, 'body')
         return Expression(_body)
 
-State.ast_type(''Expression'', 'mod', ["'body'"])
+State.ast_type('Expression', 'mod', ["'body'"])
 
 
 class Suite(mod):
@@ -299,7 +299,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(_body)
 
-State.ast_type(''Suite'', 'mod', ["'body'"])
+State.ast_type('Suite', 'mod', ["'body'"])
 
 
 class stmt(AST):
@@ -356,7 +356,7 @@
             return Continue.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected stmt node, got %T", w_node)
-State.ast_type(''stmt'', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('stmt', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class FunctionDef(stmt):
 
@@ -431,7 +431,7 @@
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
 
-State.ast_type(''FunctionDef'', 'stmt', ["'name'", "'args'", "'body'", "'decorator_list'", "'returns'"])
+State.ast_type('FunctionDef', 'stmt', ["'name'", "'args'", "'body'", "'decorator_list'", "'returns'"])
 
 
 class ClassDef(stmt):
@@ -516,7 +516,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ClassDef(_name, _bases, _keywords, _body, _decorator_list, _lineno, _col_offset)
 
-State.ast_type(''ClassDef'', 'stmt', ["'name'", "'bases'", "'keywords'", "'body'", "'decorator_list'"])
+State.ast_type('ClassDef', 'stmt', ["'name'", "'bases'", "'keywords'", "'body'", "'decorator_list'"])
 
 
 class Return(stmt):
@@ -553,7 +553,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
 
-State.ast_type(''Return'', 'stmt', ["'value'"])
+State.ast_type('Return', 'stmt', ["'value'"])
 
 
 class Delete(stmt):
@@ -595,7 +595,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Delete(_targets, _lineno, _col_offset)
 
-State.ast_type(''Delete'', 'stmt', ["'targets'"])
+State.ast_type('Delete', 'stmt', ["'targets'"])
 
 
 class Assign(stmt):
@@ -645,7 +645,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assign(_targets, _value, _lineno, _col_offset)
 
-State.ast_type(''Assign'', 'stmt', ["'targets'", "'value'"])
+State.ast_type('Assign', 'stmt', ["'targets'", "'value'"])
 
 
 class AugAssign(stmt):
@@ -698,7 +698,7 @@
         _col_offset = space.int_w(w_col_offset)
         return AugAssign(_target, _op, _value, _lineno, _col_offset)
 
-State.ast_type(''AugAssign'', 'stmt', ["'target'", "'op'", "'value'"])
+State.ast_type('AugAssign', 'stmt', ["'target'", "'op'", "'value'"])
 
 
 class For(stmt):
@@ -768,7 +768,7 @@
         _col_offset = space.int_w(w_col_offset)
         return For(_target, _iter, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type(''For'', 'stmt', ["'target'", "'iter'", "'body'", "'orelse'"])
+State.ast_type('For', 'stmt', ["'target'", "'iter'", "'body'", "'orelse'"])
 
 
 class While(stmt):
@@ -830,7 +830,7 @@
         _col_offset = space.int_w(w_col_offset)
         return While(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type(''While'', 'stmt', ["'test'", "'body'", "'orelse'"])
+State.ast_type('While', 'stmt', ["'test'", "'body'", "'orelse'"])
 
 
 class If(stmt):
@@ -892,7 +892,7 @@
         _col_offset = space.int_w(w_col_offset)
         return If(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type(''If'', 'stmt', ["'test'", "'body'", "'orelse'"])
+State.ast_type('If', 'stmt', ["'test'", "'body'", "'orelse'"])
 
 
 class With(stmt):
@@ -946,7 +946,7 @@
         _col_offset = space.int_w(w_col_offset)
         return With(_items, _body, _lineno, _col_offset)
 
-State.ast_type(''With'', 'stmt', ["'items'", "'body'"])
+State.ast_type('With', 'stmt', ["'items'", "'body'"])
 
 
 class Raise(stmt):
@@ -990,7 +990,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
 
-State.ast_type(''Raise'', 'stmt', ["'exc'", "'cause'"])
+State.ast_type('Raise', 'stmt', ["'exc'", "'cause'"])
 
 
 class Try(stmt):
@@ -1068,7 +1068,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Try(_body, _handlers, _orelse, _finalbody, _lineno, _col_offset)
 
-State.ast_type(''Try'', 'stmt', ["'body'", "'handlers'", "'orelse'", "'finalbody'"])
+State.ast_type('Try', 'stmt', ["'body'", "'handlers'", "'orelse'", "'finalbody'"])
 
 
 class Assert(stmt):
@@ -1113,7 +1113,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
 
-State.ast_type(''Assert'', 'stmt', ["'test'", "'msg'"])
+State.ast_type('Assert', 'stmt', ["'test'", "'msg'"])
 
 
 class Import(stmt):
@@ -1155,7 +1155,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Import(_names, _lineno, _col_offset)
 
-State.ast_type(''Import'', 'stmt', ["'names'"])
+State.ast_type('Import', 'stmt', ["'names'"])
 
 
 class ImportFrom(stmt):
@@ -1207,7 +1207,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ImportFrom(_module, _names, _level, _lineno, _col_offset)
 
-State.ast_type(''ImportFrom'', 'stmt', ["'module'", "'names'", "'level'"])
+State.ast_type('ImportFrom', 'stmt', ["'module'", "'names'", "'level'"])
 
 
 class Global(stmt):
@@ -1247,7 +1247,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
 
-State.ast_type(''Global'', 'stmt', ["'names'"])
+State.ast_type('Global', 'stmt', ["'names'"])
 
 
 class Nonlocal(stmt):
@@ -1287,7 +1287,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Nonlocal(_names, _lineno, _col_offset)
 
-State.ast_type(''Nonlocal'', 'stmt', ["'names'"])
+State.ast_type('Nonlocal', 'stmt', ["'names'"])
 
 
 class Expr(stmt):
@@ -1325,7 +1325,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Expr(_value, _lineno, _col_offset)
 
-State.ast_type(''Expr'', 'stmt', ["'value'"])
+State.ast_type('Expr', 'stmt', ["'value'"])
 
 
 class Pass(stmt):
@@ -1355,7 +1355,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Pass(_lineno, _col_offset)
 
-State.ast_type(''Pass'', 'stmt', [])
+State.ast_type('Pass', 'stmt', [])
 
 
 class Break(stmt):
@@ -1385,7 +1385,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Break(_lineno, _col_offset)
 
-State.ast_type(''Break'', 'stmt', [])
+State.ast_type('Break', 'stmt', [])
 
 
 class Continue(stmt):
@@ -1415,7 +1415,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Continue(_lineno, _col_offset)
 
-State.ast_type(''Continue'', 'stmt', [])
+State.ast_type('Continue', 'stmt', [])
 
 
 class expr(AST):
@@ -1484,7 +1484,7 @@
             return Const.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected expr node, got %T", w_node)
-State.ast_type(''expr'', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('expr', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class BoolOp(expr):
 
@@ -1532,7 +1532,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BoolOp(_op, _values, _lineno, _col_offset)
 
-State.ast_type(''BoolOp'', 'expr', ["'op'", "'values'"])
+State.ast_type('BoolOp', 'expr', ["'op'", "'values'"])
 
 
 class BinOp(expr):
@@ -1585,7 +1585,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BinOp(_left, _op, _right, _lineno, _col_offset)
 
-State.ast_type(''BinOp'', 'expr', ["'left'", "'op'", "'right'"])
+State.ast_type('BinOp', 'expr', ["'left'", "'op'", "'right'"])
 
 
 class UnaryOp(expr):
@@ -1630,7 +1630,7 @@
         _col_offset = space.int_w(w_col_offset)
         return UnaryOp(_op, _operand, _lineno, _col_offset)
 
-State.ast_type(''UnaryOp'', 'expr', ["'op'", "'operand'"])
+State.ast_type('UnaryOp', 'expr', ["'op'", "'operand'"])
 
 
 class Lambda(expr):
@@ -1676,7 +1676,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Lambda(_args, _body, _lineno, _col_offset)
 
-State.ast_type(''Lambda'', 'expr', ["'args'", "'body'"])
+State.ast_type('Lambda', 'expr', ["'args'", "'body'"])
 
 
 class IfExp(expr):
@@ -1730,7 +1730,7 @@
         _col_offset = space.int_w(w_col_offset)
         return IfExp(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type(''IfExp'', 'expr', ["'test'", "'body'", "'orelse'"])
+State.ast_type('IfExp', 'expr', ["'test'", "'body'", "'orelse'"])
 
 
 class Dict(expr):
@@ -1784,7 +1784,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Dict(_keys, _values, _lineno, _col_offset)
 
-State.ast_type(''Dict'', 'expr', ["'keys'", "'values'"])
+State.ast_type('Dict', 'expr', ["'keys'", "'values'"])
 
 
 class Set(expr):
@@ -1826,7 +1826,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Set(_elts, _lineno, _col_offset)
 
-State.ast_type(''Set'', 'expr', ["'elts'"])
+State.ast_type('Set', 'expr', ["'elts'"])
 
 
 class ListComp(expr):
@@ -1876,7 +1876,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ListComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type(''ListComp'', 'expr', ["'elt'", "'generators'"])
+State.ast_type('ListComp', 'expr', ["'elt'", "'generators'"])
 
 
 class SetComp(expr):
@@ -1926,7 +1926,7 @@
         _col_offset = space.int_w(w_col_offset)
         return SetComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type(''SetComp'', 'expr', ["'elt'", "'generators'"])
+State.ast_type('SetComp', 'expr', ["'elt'", "'generators'"])
 
 
 class DictComp(expr):
@@ -1984,7 +1984,7 @@
         _col_offset = space.int_w(w_col_offset)
         return DictComp(_key, _value, _generators, _lineno, _col_offset)
 
-State.ast_type(''DictComp'', 'expr', ["'key'", "'value'", "'generators'"])
+State.ast_type('DictComp', 'expr', ["'key'", "'value'", "'generators'"])
 
 
 class GeneratorExp(expr):
@@ -2034,7 +2034,7 @@
         _col_offset = space.int_w(w_col_offset)
         return GeneratorExp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type(''GeneratorExp'', 'expr', ["'elt'", "'generators'"])
+State.ast_type('GeneratorExp', 'expr', ["'elt'", "'generators'"])
 
 
 class Yield(expr):
@@ -2071,7 +2071,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
 
-State.ast_type(''Yield'', 'expr', ["'value'"])
+State.ast_type('Yield', 'expr', ["'value'"])
 
 
 class YieldFrom(expr):
@@ -2109,7 +2109,7 @@
         _col_offset = space.int_w(w_col_offset)
         return YieldFrom(_value, _lineno, _col_offset)
 
-State.ast_type(''YieldFrom'', 'expr', ["'value'"])
+State.ast_type('YieldFrom', 'expr', ["'value'"])
 
 
 class Compare(expr):
@@ -2169,7 +2169,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Compare(_left, _ops, _comparators, _lineno, _col_offset)
 
-State.ast_type(''Compare'', 'expr', ["'left'", "'ops'", "'comparators'"])
+State.ast_type('Compare', 'expr', ["'left'", "'ops'", "'comparators'"])
 
 
 class Call(expr):
@@ -2231,7 +2231,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Call(_func, _args, _keywords, _lineno, _col_offset)
 
-State.ast_type(''Call'', 'expr', ["'func'", "'args'", "'keywords'"])
+State.ast_type('Call', 'expr', ["'func'", "'args'", "'keywords'"])
 
 
 class Num(expr):
@@ -2268,7 +2268,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Num(_n, _lineno, _col_offset)
 
-State.ast_type(''Num'', 'expr', ["'n'"])
+State.ast_type('Num', 'expr', ["'n'"])
 
 
 class Str(expr):
@@ -2305,7 +2305,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Str(_s, _lineno, _col_offset)
 
-State.ast_type(''Str'', 'expr', ["'s'"])
+State.ast_type('Str', 'expr', ["'s'"])
 
 
 class Bytes(expr):
@@ -2342,7 +2342,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Bytes(_s, _lineno, _col_offset)
 
-State.ast_type(''Bytes'', 'expr', ["'s'"])
+State.ast_type('Bytes', 'expr', ["'s'"])
 
 
 class NameConstant(expr):
@@ -2379,7 +2379,7 @@
         _col_offset = space.int_w(w_col_offset)
         return NameConstant(_value, _lineno, _col_offset)
 
-State.ast_type(''NameConstant'', 'expr', ["'value'"])
+State.ast_type('NameConstant', 'expr', ["'value'"])
 
 
 class Ellipsis(expr):
@@ -2409,7 +2409,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Ellipsis(_lineno, _col_offset)
 
-State.ast_type(''Ellipsis'', 'expr', [])
+State.ast_type('Ellipsis', 'expr', [])
 
 
 class Attribute(expr):
@@ -2461,7 +2461,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
 
-State.ast_type(''Attribute'', 'expr', ["'value'", "'attr'", "'ctx'"])
+State.ast_type('Attribute', 'expr', ["'value'", "'attr'", "'ctx'"])
 
 
 class Subscript(expr):
@@ -2514,7 +2514,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
 
-State.ast_type(''Subscript'', 'expr', ["'value'", "'slice'", "'ctx'"])
+State.ast_type('Subscript', 'expr', ["'value'", "'slice'", "'ctx'"])
 
 
 class Starred(expr):
@@ -2559,7 +2559,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Starred(_value, _ctx, _lineno, _col_offset)
 
-State.ast_type(''Starred'', 'expr', ["'value'", "'ctx'"])
+State.ast_type('Starred', 'expr', ["'value'", "'ctx'"])
 
 
 class Name(expr):
@@ -2603,7 +2603,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Name(_id, _ctx, _lineno, _col_offset)
 
-State.ast_type(''Name'', 'expr', ["'id'", "'ctx'"])
+State.ast_type('Name', 'expr', ["'id'", "'ctx'"])
 
 
 class List(expr):
@@ -2652,7 +2652,7 @@
         _col_offset = space.int_w(w_col_offset)
         return List(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type(''List'', 'expr', ["'elts'", "'ctx'"])
+State.ast_type('List', 'expr', ["'elts'", "'ctx'"])
 
 
 class Tuple(expr):
@@ -2701,7 +2701,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Tuple(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type(''Tuple'', 'expr', ["'elts'", "'ctx'"])
+State.ast_type('Tuple', 'expr', ["'elts'", "'ctx'"])
 
 
 class Const(expr):
@@ -2738,7 +2738,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Const(_value, _lineno, _col_offset)
 
-State.ast_type(''Const'', 'expr', ["'value'"])
+State.ast_type('Const', 'expr', ["'value'"])
 
 
 class expr_context(AST):
@@ -2819,7 +2819,7 @@
             return Index.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected slice node, got %T", w_node)
-State.ast_type(''slice'', 'AST', None, [])
+State.ast_type('slice', 'AST', None, [])
 
 class Slice(slice):
 
@@ -2860,7 +2860,7 @@
         _step = expr.from_object(space, w_step) if w_step is not None else None
         return Slice(_lower, _upper, _step)
 
-State.ast_type(''Slice'', 'slice', ["'lower'", "'upper'", "'step'"])
+State.ast_type('Slice', 'slice', ["'lower'", "'upper'", "'step'"])
 
 
 class ExtSlice(slice):
@@ -2893,7 +2893,7 @@
         _dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(_dims)
 
-State.ast_type(''ExtSlice'', 'slice', ["'dims'"])
+State.ast_type('ExtSlice', 'slice', ["'dims'"])
 
 
 class Index(slice):
@@ -2922,7 +2922,7 @@
             raise_required_value(space, w_node, 'value')
         return Index(_value)
 
-State.ast_type(''Index'', 'slice', ["'value'"])
+State.ast_type('Index', 'slice', ["'value'"])
 
 
 class boolop(AST):
@@ -3276,7 +3276,7 @@
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
 
-State.ast_type(''comprehension'', 'AST', ["'target'", "'iter'", "'ifs'"])
+State.ast_type('comprehension', 'AST', ["'target'", "'iter'", "'ifs'"])
 
 class excepthandler(AST):
 
@@ -3292,7 +3292,7 @@
             return ExceptHandler.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected excepthandler node, got %T", w_node)
-State.ast_type(''excepthandler'', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('excepthandler', 'AST', None, ["'lineno'", "'col_offset'"])
 
 class ExceptHandler(excepthandler):
 
@@ -3345,7 +3345,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ExceptHandler(_type, _name, _body, _lineno, _col_offset)
 
-State.ast_type(''ExceptHandler'', 'excepthandler', ["'type'", "'name'", "'body'"])
+State.ast_type('ExceptHandler', 'excepthandler', ["'type'", "'name'", "'body'"])
 
 
 class arguments(AST):
@@ -3428,7 +3428,7 @@
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
 
-State.ast_type(''arguments'', 'AST', ["'args'", "'vararg'", "'kwonlyargs'", "'kw_defaults'", "'kwarg'", "'defaults'"])
+State.ast_type('arguments', 'AST', ["'args'", "'vararg'", "'kwonlyargs'", "'kw_defaults'", "'kwarg'", "'defaults'"])
 
 class arg(AST):
 
@@ -3462,7 +3462,7 @@
         _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
         return arg(_arg, _annotation)
 
-State.ast_type(''arg'', 'AST', ["'arg'", "'annotation'"])
+State.ast_type('arg', 'AST', ["'arg'", "'annotation'"])
 
 class keyword(AST):
 
@@ -3495,7 +3495,7 @@
             raise_required_value(space, w_node, 'value')
         return keyword(_arg, _value)
 
-State.ast_type(''keyword'', 'AST', ["'arg'", "'value'"])
+State.ast_type('keyword', 'AST', ["'arg'", "'value'"])
 
 class alias(AST):
 
@@ -3527,7 +3527,7 @@
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
-State.ast_type(''alias'', 'AST', ["'name'", "'asname'"])
+State.ast_type('alias', 'AST', ["'name'", "'asname'"])
 
 class withitem(AST):
 
@@ -3562,7 +3562,7 @@
         _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
         return withitem(_context_expr, _optional_vars)
 
-State.ast_type(''withitem'', 'AST', ["'context_expr'", "'optional_vars'"])
+State.ast_type('withitem', 'AST', ["'context_expr'", "'optional_vars'"])
 
 class ASTVisitor(object):
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -101,7 +101,7 @@
                           % (typ.name,), 3)
             self.emit("raise oefmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
-            self.emit("State.ast_type('%r', 'AST', None, %s)" %
+            self.emit("State.ast_type(%r, 'AST', None, %s)" %
                       (base, [repr(attr.name) for attr in sum.attributes]))
             self.emit("")
             for cons in sum.types:
@@ -118,7 +118,7 @@
         self.emit("visitor.visit_%s(self)" % (name,), 2)
         self.emit("")
         self.make_converters(product.fields, name)
-        self.emit("State.ast_type('%r', 'AST', %s)" %
+        self.emit("State.ast_type(%r, 'AST', %s)" %
                   (name, [repr(f.name) for f in product.fields]))
         self.emit("")
 
@@ -262,7 +262,7 @@
         self.emit("")
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
-        self.emit("State.ast_type('%r', '%s', %s)" % 
+        self.emit("State.ast_type(%r, '%s', %s)" % 
                   (cons.name, base, [repr(f.name) for f in cons.fields]))
         self.emit("")
 

From pypy.commits at gmail.com  Thu Jun  9 13:27:04 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 09 Jun 2016 10:27:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix Matmul from python3.5 diff (matmul ->
 matmult)
Message-ID: <5759a6e8.665ec20a.bcd5e.ffffe298@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85060:afe63cdf3335
Date: 2016-06-09 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/afe63cdf3335/

Log:	Fix Matmul from python3.5 diff (matmul -> matmult)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -17,7 +17,7 @@
     '/='  : ast.Div,
     '//=' : ast.FloorDiv,
     '%='  : ast.Mod,
-    '@='  : ast.MatMul,
+    '@='  : ast.MatMult,
     '<<='  : ast.LShift,
     '>>='  : ast.RShift,
     '&='  : ast.BitAnd,
@@ -39,7 +39,7 @@
     tokens.SLASH : ast.Div,
     tokens.DOUBLESLASH : ast.FloorDiv,
     tokens.PERCENT : ast.Mod,
-    tokens.AT : ast.MatMul
+    tokens.AT : ast.MatMult
 })
 
 
@@ -533,8 +533,7 @@
         # and varargslist (lambda definition).
         if arguments_node.type == syms.parameters:
             if arguments_node.num_children() == 2:
-                return ast.arguments(None, None, None, None, None, None, None,
-                                     None)
+                return ast.arguments(None, None, None, None, None, None)
             arguments_node = arguments_node.get_child(1)
         i = 0
         child_count = arguments_node.num_children()
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -66,7 +66,7 @@
     ast.BitAnd: ops.BINARY_AND,
     ast.BitXor: ops.BINARY_XOR,
     ast.FloorDiv: ops.BINARY_FLOOR_DIVIDE,
-    ast.MatMul: ops.BINARY_MATRIX_MULTIPLY
+    ast.MatMult: ops.BINARY_MATRIX_MULTIPLY
 })
 
 inplace_operations = misc.dict_to_switch({
@@ -82,7 +82,7 @@
     ast.BitAnd: ops.INPLACE_AND,
     ast.BitXor: ops.INPLACE_XOR,
     ast.FloorDiv: ops.INPLACE_FLOOR_DIVIDE,
-    ast.MatMul: ops.INPLACE_MATRIX_MULTIPLY
+    ast.MatMult: ops.INPLACE_MATRIX_MULTIPLY
 })
 
 compare_operations = misc.dict_to_switch({
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -134,7 +134,7 @@
     ast.BitOr : _binary_fold("or_"),
     ast.BitXor : _binary_fold("xor"),
     ast.BitAnd : _binary_fold("and_"),
-    ast.MatMul : _binary_fold("matmul"),
+    ast.MatMult : _binary_fold("matmul"),
 }
 unrolling_binary_folders = unrolling_iterable(binary_folders.items())
 

From pypy.commits at gmail.com  Thu Jun  9 13:42:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 09 Jun 2016 10:42:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add include guards to pymem.h: fixes
 issue #2321
Message-ID: <5759aa81.094ac20a.6bfe0.ffffe705@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r85061:68486f0f79c6
Date: 2016-06-09 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/68486f0f79c6/

Log:	Add include guards to pymem.h: fixes issue #2321

diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */

From pypy.commits at gmail.com  Thu Jun  9 14:56:53 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 09 Jun 2016 11:56:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Delete deprecated arguments and reorder
 returned arguments in handle_arguments
Message-ID: <5759bbf5.832c1c0a.20d42.27d9@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85062:61e3f61af25a
Date: 2016-06-09 20:55 +0200
http://bitbucket.org/pypy/pypy/changeset/61e3f61af25a/

Log:	Delete deprecated arguments and reorder returned arguments in
	handle_arguments

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -627,8 +627,8 @@
                 i += 3
             else:
                 raise AssertionError("unknown node in argument list")
-        return ast.arguments(pos, vararg, varargann, kwonly, kwarg,
-                             kwargann, posdefaults, kwdefaults)
+        return ast.arguments(pos, vararg, kwonly, kwdefaults, kwarg,
+                             posdefaults)
 
     def handle_keywordonly_args(self, arguments_node, i, kwonly, kwdefaults):
         if kwonly is None:

From pypy.commits at gmail.com  Thu Jun  9 18:13:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Jun 2016 15:13:51 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Pipes and forked subprocesses
Message-ID: <5759ea1f.4fa51c0a.1b9a8.7b12@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85063:502859b18e51
Date: 2016-06-10 00:14 +0200
http://bitbucket.org/pypy/pypy/changeset/502859b18e51/

Log:	Pipes and forked subprocesses

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -71,16 +71,16 @@
 /* ------------------------------------------------------------ */
 
 
-static void write_all(int fd, const char *buf, ssize_t count)
+static void write_all(const void *buf, ssize_t count)
 {
     while (count > 0) {
-        ssize_t wsize = write(fd, buf, count);
+        ssize_t wsize = write(rpy_rev_fileno, buf, count);
         if (wsize <= 0) {
             if (wsize == 0)
-                fprintf(stderr, "Writing to PYPYREVDB file: "
+                fprintf(stderr, "writing to RevDB file: "
                                 "unexpected non-blocking mode\n");
             else
-                fprintf(stderr, "Fatal error: writing to PYPYREVDB file: %m\n");
+                fprintf(stderr, "Fatal error: writing to RevDB file: %m\n");
             abort();
         }
         buf += wsize;
@@ -114,7 +114,7 @@
     h.version = RDB_VERSION;
     h.argc = argc;
     h.argv = argv;
-    write_all(rpy_rev_fileno, (const char *)&h, sizeof(h));
+    write_all((const char *)&h, sizeof(h));
 }
 
 RPY_EXTERN
@@ -128,7 +128,7 @@
     if (size == 0 || rpy_rev_fileno < 0)
         return;
 
-    write_all(rpy_rev_fileno, rpy_rev_buffer, size);
+    write_all(rpy_rev_buffer, size);
 }
 
 
@@ -182,24 +182,44 @@
    writing more data.
 */
 
-#define FROZEN_PROCESSES   30
-#define GOLDEN_RATIO       0.618034
+#define NUM_FROZEN_PROCESSES   30
+#define GOLDEN_RATIO           0.618034
+
+#define RD_SIDE   0
+#define WR_SIDE   1
+
+static int frozen_num_pipes = 0;
+static int frozen_pipes[NUM_FROZEN_PROCESSES][2];
+static uint64_t frozen_time[NUM_FROZEN_PROCESSES];
+static int frozen_pipe_signal[2];
+
+enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
+static int process_kind = PK_MAIN_PROCESS;
 
 static uint64_t total_stop_points;
 
 
-static ssize_t read_at_least(int fd, char *buf,
-                             ssize_t count_min, ssize_t count_max)
+static void attach_gdb(void)
+{
+    char cmdline[80];
+    sprintf(cmdline, "term -c \"gdb --pid=%d\"", getpid());
+    system(cmdline);
+    sleep(1);
+}
+
+static ssize_t read_at_least(void *buf, ssize_t count_min, ssize_t count_max)
 {
     ssize_t result = 0;
     assert(count_min <= count_max);
     while (result < count_min) {
-        ssize_t rsize = read(fd, buf + result, count_max - result);
+        ssize_t rsize = read(rpy_rev_fileno, buf + result, count_max - result);
         if (rsize <= 0) {
             if (rsize == 0)
-                fprintf(stderr, "RevDB file appears truncated\n");
+                fprintf(stderr, "[%d] RevDB file appears truncated\n",
+                        process_kind);
             else
-                fprintf(stderr, "RevDB file read error: %m\n");
+                fprintf(stderr, "[%d] RevDB file read error: %m\n",
+                        process_kind);
             exit(1);
         }
         result += rsize;
@@ -207,9 +227,9 @@
     return result;
 }
 
-static void read_all(int fd, char *buf, ssize_t count)
+static void read_all(void *buf, ssize_t count)
 {
-    (void)read_at_least(fd, buf, count, count);
+    (void)read_at_least(buf, count, count);
 }
 
 static void setup_replay_mode(int *argc_p, char **argv_p[])
@@ -233,7 +253,7 @@
 
     assert(RPY_RDB_REPLAY == 1);
 
-    read_all(rpy_rev_fileno, (char *)&h, sizeof(h));
+    read_all(&h, sizeof(h));
 
     if (h.signature != RDB_SIGNATURE) {
         fprintf(stderr, "'%s' is not a RevDB file (or wrong platform)\n",
@@ -259,6 +279,11 @@
     rpy_revdb.buf_p = rpy_rev_buffer;
     rpy_revdb.buf_limit = rpy_rev_buffer;
     rpy_revdb.stop_point_break = 1;
+
+    if (pipe(frozen_pipe_signal) < 0) {
+        perror("pipe");
+        exit(1);
+    }
 }
 
 RPY_EXTERN
@@ -267,8 +292,7 @@
     ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p;
     assert(keep >= 0);
     memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep);
-    rsize = read_at_least(rpy_rev_fileno,
-                          rpy_rev_buffer + keep,
+    rsize = read_at_least(rpy_rev_buffer + keep,
                           expected_size - keep,
                           sizeof(rpy_rev_buffer) - keep);
     rpy_revdb.buf_p = rpy_rev_buffer;
@@ -276,9 +300,82 @@
     return rpy_rev_buffer;
 }
 
+struct action_s {
+    const char *name;
+    void (*act)(char *);
+};
+
+static void process_input(struct action_s actions[])
+{
+    char input[256], *p;
+    struct action_s *a;
+
+    if (fgets(input, sizeof(input), stdin) != input) {
+        fprintf(stderr, "\n");
+        strcpy(input, "exit");
+    }
+
+    p = input;
+    while (*p > ' ')
+        p++;
+    if (*p != 0) {
+        *p = 0;
+        p++;
+    }
+    a = actions;
+    while (a->name != NULL && strcmp(a->name, input) != 0) {
+        a++;
+    }
+    if (a->name != NULL) {
+        a->act(p);
+    }
+    else if (strcmp(input, "help") == 0) {
+        a = actions;
+        printf("available commands:\n");
+        while (a->name != NULL) {
+            printf("\t%s\n", a->name);
+            a++;
+        }
+    }
+    else if (input[0] != 0) {
+        printf("bad command '%s', try 'help'\n", input);
+    }
+}
+
+static int read_pipe(int fd, void *buf, ssize_t count)
+{
+    while (count > 0) {
+        ssize_t got = read(fd, buf, count);
+        if (got <= 0)
+            return -1;
+        count -= got;
+        buf += got;
+    }
+    return 0;
+}
+
+static int write_pipe(int fd, const void *buf, ssize_t count)
+{
+    while (count > 0) {
+        ssize_t wrote = write(fd, buf, count);
+        if (wrote <= 0)
+            return -1;
+        count -= wrote;
+        buf += wrote;
+    }
+    return 0;
+}
+
 static void check_at_end(uint64_t stop_points)
 {
     char dummy[1];
+    uint64_t target_time;
+
+    if (process_kind != PK_MAIN_PROCESS) {
+        fprintf(stderr, "[%d] Unexpectedly falling off the end\n",
+                process_kind);
+        exit(1);
+    }
     if (stop_points != rpy_revdb.stop_point_seen) {
         fprintf(stderr, "Bad number of stop points\n");
         exit(1);
@@ -292,15 +389,176 @@
         fprintf(stderr, "RevDB file modified while reading?\n");
         exit(1);
     }
-    printf("Replaying finished, %lld stop points\n", (long long)stop_points);
+    if (frozen_num_pipes == 0) {
+        fprintf(stderr, "RevDB file does not contain any stop points\n");
+        exit(1);
+    }
+
+    printf("Replaying finished\n");
+    printf("stop_points=%lld\n", (long long)stop_points);
+
+    close(frozen_pipe_signal[WR_SIDE]);
+    frozen_pipe_signal[WR_SIDE] = -1;
+
+    target_time = frozen_time[frozen_num_pipes-1];
+    while (target_time != (uint64_t)-1) {
+        int p = frozen_num_pipes - 1;
+        if (target_time > frozen_time[p])
+            target_time = frozen_time[p];
+        while (frozen_time[p] > target_time)
+            p--;
+        if (write_pipe(frozen_pipes[p][WR_SIDE],
+                       &target_time, sizeof(target_time)) < 0) {
+            fprintf(stderr, "broken pipe to frozen subprocess\n");
+            exit(1);
+        }
+        /* blocking here while the p'th frozen process spawns a debug process
+           and the user interacts with it; then: */
+        if (read_pipe(frozen_pipe_signal[RD_SIDE], &target_time,
+                      sizeof(target_time)) < 0) {
+            fprintf(stderr, "broken signal pipe\n");
+            exit(1);
+        }
+    }
     exit(0);
 }
 
+static void run_frozen_process(int frozen_pipe_fd)
+{
+    uint64_t target_time;
+    pid_t child_pid;
+
+    while (1) {
+        if (read_pipe(frozen_pipe_fd, &target_time, sizeof(target_time)) < 0)
+            exit(1);
+
+        child_pid = fork();
+        if (child_pid == -1) {
+            perror("fork");
+            exit(1);
+        }
+        if (child_pid == 0) {
+            /* in the child: this is a debug process */
+            process_kind = PK_DEBUG_PROCESS;
+            assert(target_time >= rpy_revdb.stop_point_seen);
+            rpy_revdb.stop_point_break = target_time;
+            /* continue "running" the RPython program until we reach
+               exactly the specified target_time */
+            break;
+        }
+        else {
+            /* in the parent: the frozen process, which waits for
+               the debug process to finish to reclaim the pid,
+               and then loops to wait for the next wake-up */
+            int status;
+            waitpid(child_pid, &status, 0);
+            if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
+                ;     /* normal exit */
+            else {
+                target_time = (uint64_t)-1;
+                fprintf(stderr, "debugging subprocess died\n");
+                write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
+                           sizeof(target_time));
+                exit(1);    /* error */
+            }
+        }
+    }
+}
+
+static void make_new_frozen_process(void)
+{
+    pid_t child_pid;
+    int *fds;
+    off_t fileno_offset;
+
+    if (frozen_num_pipes >= NUM_FROZEN_PROCESSES) {
+        fprintf(stderr, "stop_point_break overflow?\n");
+        exit(1);
+    }
+
+    fprintf(stderr, "forking at time %llu\n",
+            (unsigned long long)rpy_revdb.stop_point_seen);
+
+    fds = frozen_pipes[frozen_num_pipes];
+    if (pipe(fds) < 0) {
+        perror("pipe");
+        exit(1);
+    }
+    frozen_time[frozen_num_pipes] = rpy_revdb.stop_point_seen;
+    frozen_num_pipes += 1;
+
+    fileno_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR);
+
+    child_pid = fork();
+    if (child_pid == -1) {
+        perror("fork");
+        exit(1);
+    }
+    if (child_pid == 0) {
+        /* in the child: this is a frozen process */
+        process_kind = PK_FROZEN_PROCESS;
+        close(fds[WR_SIDE]);
+        fds[WR_SIDE] = -1;
+        run_frozen_process(fds[RD_SIDE]);
+        /* when we reach that point, we are in the debugging process */
+        lseek(rpy_rev_fileno, fileno_offset, SEEK_SET);
+    }
+    else {
+        /* in the main process: continue reloading the revdb log */
+        uint64_t delta = total_stop_points - rpy_revdb.stop_point_break;
+        delta = (uint64_t)(delta * (1 - GOLDEN_RATIO));
+        if (delta == 0)
+            rpy_revdb.stop_point_break = total_stop_points;
+        else
+            rpy_revdb.stop_point_break += delta;
+        close(fds[RD_SIDE]);
+        fds[RD_SIDE] = -1;
+    }
+}
+
+static void act_exit(char *p)
+{
+    uint64_t target_time = (uint64_t)-1;
+    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
+               sizeof(target_time));
+    exit(0);
+}
+
+static void act_go(char *p)
+{
+    uint64_t target_time = strtoull(p, NULL, 10);
+    if (target_time == 0) {
+        printf("usage: go \n");
+        return;
+    }
+    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
+               sizeof(target_time));
+    exit(0);
+}
+
+static void run_debug_process(void)
+{
+    static struct action_s actions_1[] = {
+        { "go", act_go },
+        { "exit", act_exit },
+        { NULL }
+    };
+    while (1) {
+        printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
+        fflush(stdout);
+        process_input(actions_1);
+    }
+}
+
 RPY_EXTERN
 void rpy_reverse_db_break(long stop_point)
 {
-    printf("break #%ld after %lld stop points\n", stop_point,
-           (long long)rpy_revdb.stop_point_seen);
+    if (process_kind == PK_MAIN_PROCESS)
+        make_new_frozen_process();
+
+    if (process_kind == PK_DEBUG_PROCESS)
+        if (rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)
+            run_debug_process();
 }
 
 

From pypy.commits at gmail.com  Fri Jun 10 04:54:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 01:54:38 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: tweaks
Message-ID: <575a804e.cdcf1c0a.2514f.ffff8e83@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85065:aff490e57fe0
Date: 2016-06-10 10:30 +0200
http://bitbucket.org/pypy/pypy/changeset/aff490e57fe0/

Log:	tweaks

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -5,6 +5,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "rdb-src/revdb_include.h"
 
@@ -107,14 +108,14 @@
             abort();
         }
         atexit(rpy_reverse_db_flush);
+
+        memset(&h, 0, sizeof(h));
+        h.signature = RDB_SIGNATURE;
+        h.version = RDB_VERSION;
+        h.argc = argc;
+        h.argv = argv;
+        write_all((const char *)&h, sizeof(h));
     }
-
-    memset(&h, 0, sizeof(h));
-    h.signature = RDB_SIGNATURE;
-    h.version = RDB_VERSION;
-    h.argc = argc;
-    h.argv = argv;
-    write_all((const char *)&h, sizeof(h));
 }
 
 RPY_EXTERN
@@ -184,6 +185,7 @@
 
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
 static int process_kind = PK_MAIN_PROCESS;
+static uint64_t latest_fork;
 
 static uint64_t total_stop_points;
 
@@ -294,18 +296,16 @@
     void (*act)(char *);
 };
 
-static void process_input(struct action_s actions[])
+static void process_input(char *input, const char *kind,
+                          struct action_s actions[])
 {
-    char input[256], *p;
+    char *p;
     struct action_s *a;
 
-    if (fgets(input, sizeof(input), stdin) != input) {
-        fprintf(stderr, "\n");
-        strcpy(input, "exit");
-    }
-
+    while (isspace(*input))
+        input++;
     p = input;
-    while (*p > ' ')
+    while (*p != 0 && !isspace(*p))
         p++;
     if (*p != 0) {
         *p = 0;
@@ -320,14 +320,15 @@
     }
     else if (strcmp(input, "help") == 0) {
         a = actions;
-        printf("available commands:\n");
+        printf("select %s:\n", kind);
         while (a->name != NULL) {
-            printf("\t%s\n", a->name);
+            if (*a->name != 0)
+                printf("\t%s\n", a->name);
             a++;
         }
     }
-    else if (input[0] != 0) {
-        printf("bad command '%s', try 'help'\n", input);
+    else {
+        printf("bad %s '%s', try 'help'\n", kind, input);
     }
 }
 
@@ -430,6 +431,7 @@
             /* in the child: this is a debug process */
             process_kind = PK_DEBUG_PROCESS;
             assert(target_time >= rpy_revdb.stop_point_seen);
+            latest_fork = rpy_revdb.stop_point_seen;
             rpy_revdb.stop_point_break = target_time;
             /* continue "running" the RPython program until we reach
                exactly the specified target_time */
@@ -505,7 +507,7 @@
     }
 }
 
-static void act_exit(char *p)
+static void act_quit(char *p)
 {
     uint64_t target_time = (uint64_t)-1;
     write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
@@ -525,17 +527,42 @@
     exit(0);
 }
 
+static void act_info_fork(char *p)
+{
+    printf("latest_fork=%llu\n", (unsigned long long)latest_fork);
+}
+
+static void act_info(char *p)
+{
+    static struct action_s actions_info[] = {
+        { "fork", act_info_fork },
+        { NULL }
+    };
+    process_input(p, "category", actions_info);
+}
+
+static void act_nop(char *p)
+{
+}
+
 static void run_debug_process(void)
 {
     static struct action_s actions_1[] = {
         { "go", act_go },
-        { "exit", act_exit },
+        { "info", act_info },
+        { "quit", act_quit },
+        { "", act_nop },
         { NULL }
     };
     while (1) {
+        char input[256];
         printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
         fflush(stdout);
-        process_input(actions_1);
+        if (fgets(input, sizeof(input), stdin) != input) {
+            fprintf(stderr, "\n");
+            strcpy(input, "exit");
+        }
+        process_input(input, "command", actions_1);
     }
 }
 

From pypy.commits at gmail.com  Fri Jun 10 04:54:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 01:54:40 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix and improve tests
Message-ID: <575a8050.56311c0a.a030.ffff8d2a@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85066:1a7598c93c56
Date: 2016-06-10 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/1a7598c93c56/

Log:	Fix and improve tests

diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -1,6 +1,6 @@
 import py
 import os, sys
-import array, struct
+import re, array, struct
 from rpython.tool.udir import udir
 from rpython.translator.interactive import Translation
 from rpython.rlib.rarithmetic import LONG_BIT
@@ -32,10 +32,11 @@
         return self.cur == len(self.buffer)
 
 
-class TestBasic(object):
+class BaseTests(object):
 
-    def getcompiled(self, entry_point, argtypes, backendopt=True):
+    def compile(self, entry_point, argtypes, backendopt=True):
         t = Translation(entry_point, None, gc="boehm")
+        self.t = t
         t.config.translation.reverse_debugger = True
         t.config.translation.rweakref = False
         if not backendopt:
@@ -48,29 +49,27 @@
         self.rdbname = os.path.join(os.path.dirname(str(self.exename)),
                                     'log.rdb')
 
-        def run(*argv):
-            env = os.environ.copy()
-            env['PYPYREVDB'] = self.rdbname
-            stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
-                                                       expect_crash=9)
-            print >> sys.stderr, stderr
-            return stdout
-
-        def replay():
-            stdout = t.driver.cbuilder.cmdexec("--replay '%s'" % self.rdbname)
-            return stdout
-
-        return run, replay
+    def run(self, *argv):
+        env = os.environ.copy()
+        env['PYPYREVDB'] = self.rdbname
+        t = self.t
+        stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
+                                                   expect_crash=9)
+        print >> sys.stderr, stderr
+        return stdout
 
     def fetch_rdb(self):
         return RDB(self.rdbname)
 
+
+class TestRecording(BaseTests):
+
     def test_simple(self):
         def main(argv):
             print argv[1:]
             return 9
-        fn, replay = self.getcompiled(main, [], backendopt=False)
-        assert fn('abc d') == '[abc, d]\n'
+        self.compile(main, [], backendopt=False)
+        assert self.run('abc d') == '[abc, d]\n'
         rdb = self.fetch_rdb()
         assert rdb.argc == 3
         #
@@ -98,10 +97,21 @@
         assert rdb.done()
         #
         assert got == [self.exename, 'abc', 'd']
-        #
-        # Now try the replay mode (just "doesn't crash" for now)
-        out = replay()
-        assert out == "Replaying finished, 0 stop points\n"
+
+
+class TestInteraction(BaseTests):
+    """
+    These tests require pexpect (UNIX-only).
+    http://pexpect.sourceforge.net/
+    """
+    def replay(self, **kwds):
+        import pexpect
+        self.EOF = pexpect.EOF
+        kwds.setdefault('timeout', 10)
+        child = pexpect.spawn(str(self.exename),
+                              ['--replay', str(self.rdbname)], **kwds)
+        child.logfile = sys.stdout
+        return child
 
     def test_simple_interpreter(self):
         def main(argv):
@@ -109,9 +119,30 @@
                 revdb.stop_point(42)
                 print op
             return 9
-        fn, replay = self.getcompiled(main, [], backendopt=False)
-        assert fn('abc d ef') == 'abc\nd\nef\n'
+        self.compile(main, [], backendopt=False)
+        assert self.run('abc d ef') == 'abc\nd\nef\n'
         assert self.fetch_rdb().number_of_stop_points() == 3
-        out = replay()
-        assert out == ("break #42 after 1 stop points\n"
-                       "Replaying finished, 3 stop points\n")
+
+        child = self.replay()
+
+        def wait(s):
+            child.expect(re.escape(s))
+
+        wait('stop_points=3\r\n')
+        wait('(3)$ ')
+        child.sendline('go 1')
+        wait('(1)$ ')
+        child.sendline('')
+        wait('(1)$ ')
+        child.sendline('go 52')
+        wait('(3)$ ')
+        child.sendline('help')
+        wait('select command:\r\n')
+        # ...
+        wait('(3)$ ')
+        child.sendline('info')
+        wait("bad category '', try 'help'\r\n")
+        child.sendline('info fork')
+        wait('latest_fork=3\r\n')
+        child.sendline('quit')
+        child.expect(self.EOF)

From pypy.commits at gmail.com  Fri Jun 10 04:54:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 01:54:36 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Document the simplified model
 actually implemented
Message-ID: <575a804c.0e9e1c0a.42103.1e85@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85064:32ee948070b4
Date: 2016-06-10 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/32ee948070b4/

Log:	Document the simplified model actually implemented

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -154,32 +154,21 @@
         ...
         [frozen process n]
 
-   When all frozen processes are made, the main process enters
-   interactive mode.  In interactive mode, the main process reads from
-   stdin a version number to go to.  It picks the correct frozen
-   process (the closest one that is before in time); let's say it is
-   process #p.  It sends the version number to it by writing to pipe
-   #p.  The corresponding frozen process wakes up, and forks again
-   into a debugging process.  The main and the frozen process then
-   block.
+   The main process's job, once reloading is finished, is only to
+   active debugging processes, one at a time.  To go to a specific
+   target time, it activates the right frozen process by sending
+   'target_time' over the corresponding pipe.  The frozen process
+   forks the debugging process, and the debugging process goes forward
+   until it reaches 'target_time'.
 
-   The debugging process first goes forward in time until it reaches
-   the right version number.  Then it interacts with the user (or a
-   pdb-like program outside) on stdin/stdout.  This goes on until an
-   "exit" command is received and the debugging process dies.  At that
-   point its parent (the frozen process) continues and signals its own
-   parent (the main process) by writing to a separate signalling pipe.
-   The main process then wakes up again, and the loop closes: it reads
-   on stdin the next version number that we're interested in, and asks
-   the right frozen process to make a debugging process again.
+   The debugging process is then interacting with the user on
+   stdin/stdout.
 
-   Note how we have, over time, several processes that read and
-   process stdin; it should work because they are strictly doing that
-   in sequence, never concurrently.  To avoid the case where stdin is
-   buffered inside one process but a different process should read it,
-   we write markers to stdout when such switches occur.  The outside
-   controlling program must wait until it sees these markers before
-   writing more data.
+   A few commands like 'go ' will cause the debugging
+   process to send the 'target_time' back over a signalling pipe to
+   the main process, and then finish.  The main process receives that
+   'target_time', and the loop closes: it activates the right frozen
+   process, which will go forward and re-enter interactive mode.
 */
 
 #define NUM_FROZEN_PROCESSES   30

From pypy.commits at gmail.com  Fri Jun 10 05:30:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:30:52 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-openbsd: Close branch vmprof-openbsd
Message-ID: <575a88cc.665ec20a.bcd5e.fffffc4e@mx.google.com>

Author: Armin Rigo 
Branch: vmprof-openbsd
Changeset: r85067:3fb284829033
Date: 2016-06-10 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/3fb284829033/

Log:	Close branch vmprof-openbsd


From pypy.commits at gmail.com  Fri Jun 10 05:31:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:31:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in vmprof-openbsd (pull request
 #456)
Message-ID: <575a890c.4a9bc20a.b68d3.ffff9737@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85068:2c0977d6305f
Date: 2016-06-10 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2c0977d6305f/

Log:	Merged in vmprof-openbsd (pull request #456)

	Make VMProf work on OpenBSD

diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,14 @@
+#if !defined(__OpenBSD__)
 #define HAVE_SYS_UCONTEXT_H
+#endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,8 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(__OpenBSD__)
+#include 
 #endif
 
 

From pypy.commits at gmail.com  Fri Jun 10 05:35:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:35:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: move __OpenBSD__ checks so that they
 occur only in vmprof_config.h
Message-ID: <575a89e2.831dc20a.d0f32.fffffd84@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85069:15ff5b69d565
Date: 2016-06-10 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/15ff5b69d565/

Log:	move __OpenBSD__ checks so that they occur only in vmprof_config.h

diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,6 +1,9 @@
 #if !defined(__OpenBSD__)
-#define HAVE_SYS_UCONTEXT_H
+#  define HAVE_SYS_UCONTEXT_H
+#else
+#  define HAVE_SIGNAL_H
 #endif
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,8 +65,10 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
-#elif defined(__OpenBSD__)
+#elif defined(HAVE_SIGNAL_H)
 #include 
+#else
+#  error "don't know how to get the pc on this platform"
 #endif
 
 

From pypy.commits at gmail.com  Fri Jun 10 05:35:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:35:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: ignore branches
Message-ID: <575a89fc.011f1c0a.e6649.30e3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85070:9140946a631f
Date: 2016-06-10 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/9140946a631f/

Log:	ignore branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -19,3 +19,6 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd

From pypy.commits at gmail.com  Fri Jun 10 05:51:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:51:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <575a8da6.e778c20a.e1927.ffffff32@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85071:3586ac55f668
Date: 2016-06-10 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/3586ac55f668/

Log:	fix test

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -554,6 +554,7 @@
         assert res # we optimized it
         assert hdr_dst.tid & minimark.GCFLAG_TRACK_YOUNG_PTRS == 0 # and we copied the flag
         #
+        self.gc.card_page_indices = 128     # force > 0
         hdr_src.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_dst.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_src.tid |= minimark.GCFLAG_HAS_CARDS

From pypy.commits at gmail.com  Fri Jun 10 05:51:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 02:51:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Argh, nonsense
Message-ID: <575a8da8.4aa71c0a.9c37.37a3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85072:7686902f6a0c
Date: 2016-06-10 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7686902f6a0c/

Log:	Argh, nonsense

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -61,6 +61,7 @@
     name = options.name
     if not name:
         name = 'pypy-nightly'
+    assert '/' not in name
     rename_pypy_c = options.pypy_c
     override_pypy_c = options.override_pypy_c
 
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -71,7 +71,6 @@
         builddir = udir.ensure("build", dir=True)
         retval, builddir = package.package(
             '--without-cffi', '--builddir', str(builddir),
-            str(py.path.local(pypydir).dirpath()),
             test, self.rename_pypy_c, _fake=True)
 
     def test_with_zipfile_module(self):

From pypy.commits at gmail.com  Fri Jun 10 06:12:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 03:12:48 -0700 (PDT)
Subject: [pypy-commit] buildbot default: Fix buildbot to use (and require)
 the keyword argument version of
Message-ID: <575a92a0.c61f1c0a.e4edd.2ac9@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r1010:1c7ca3390961
Date: 2016-06-10 12:14 +0200
http://bitbucket.org/pypy/buildbot/changeset/1c7ca3390961/

Log:	Fix buildbot to use (and require) the keyword argument version of
	package.py, in order to drop backward compatibility

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -606,7 +606,8 @@
             description="compress pypy-c",
             haltOnFailure=False,
             command=prefix + ["python", "pypy/tool/release/package.py",
-                              "--targetdir=.", ".", WithProperties(name)],
+                              "--targetdir=.",
+                              "--archive-name", WithProperties(name)],
             workdir='build'))
         nightly = '~/nightly/'
         extension = get_extension(platform)
@@ -734,7 +735,8 @@
         self.addStep(ShellCmd(
             description="compress pypy-c",
             command=prefix + ["python", "pypy/tool/release/package.py",
-                              "--targetdir=.", ".", WithProperties(name)],
+                              "--targetdir=.",
+                              "--archive-name", WithProperties(name)],
             haltOnFailure=True,
             workdir='build'))
         nightly = '~/nightly/'

From pypy.commits at gmail.com  Fri Jun 10 06:20:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 03:20:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Get rid of the backward-compatibility
 of positional arguments.
Message-ID: <575a9487.0e9e1c0a.42103.42c3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85073:24501b6937bb
Date: 2016-06-10 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/24501b6937bb/

Log:	Get rid of the backward-compatibility of positional arguments.

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -3,10 +3,12 @@
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--options] pypy-VER-PLATFORM
+    package.py [--options] --archive-name=pypy-VER-PLATFORM
 
 The output is found in the directory from --builddir,
 by default /tmp/usession-YOURNAME/build/.
+
+For a list of all options, see 'package.py --help'.
 """
 
 import shutil
@@ -289,26 +291,12 @@
         help='destination dir for archive')
     parser.add_argument('--override_pypy_c', type=str, default='',
         help='use as pypy exe instead of pypy/goal/pypy-c')
-    # Positional arguments, for backward compatability with buldbots
-    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
-        metavar='[archive-name] [rename_pypy_c] [targetdir] [override_pypy_c]',
-        )
     options = parser.parse_args(args)
 
-    # Handle positional arguments, choke if both methods are used
-    for i,target, default in ([0, 'name', ''], [1, 'pypy_c', pypy_exe],
-                              [2, 'targetdir', ''], [3,'override_pypy_c', '']):
-        if len(options.extra_args)>i:
-            if getattr(options, target) != default:
-                print 'positional argument',i,target,'already has value',getattr(options, target)
-                parser.print_help()
-                return
-            setattr(options, target, options.extra_args[i])
     if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
         options.nostrip = True
-
     if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        options.tk = True
+        options.no_tk = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -22,7 +22,9 @@
     def test_dir_structure(self, test='test'):
         retval, builddir = package.package(
             '--without-cffi',
-            test, self.rename_pypy_c, _fake=True)
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
         assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d.%d' % CPYTHON_VERSION[:2]
@@ -71,7 +73,9 @@
         builddir = udir.ensure("build", dir=True)
         retval, builddir = package.package(
             '--without-cffi', '--builddir', str(builddir),
-            test, self.rename_pypy_c, _fake=True)
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
 
     def test_with_zipfile_module(self):
         prev = package.USE_ZIPFILE_MODULE

From pypy.commits at gmail.com  Fri Jun 10 08:46:24 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 10 Jun 2016 05:46:24 -0700 (PDT)
Subject: [pypy-commit] pypy hypothesis-apptest: try to make it possible to
 test applevel functions (and soon methods) with
Message-ID: <575ab6a0.24f9c20a.1929.607a@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: hypothesis-apptest
Changeset: r85074:1fb5d21e02ba
Date: 2016-06-09 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/1fb5d21e02ba/

Log:	try to make it possible to test applevel functions (and soon
	methods) with hypothesis. Limited to one argument for now.

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1121,7 +1121,7 @@
     if not isinstance(source, str):
         flags = source.__code__.co_flags
         source = py.std.inspect.getsource(source).lstrip()
-        while source.startswith(('@py.test.mark.', '@pytest.mark.')):
+        while source.startswith(('@py.test.mark.', '@pytest.mark.', '@app_hypothesis_given')):
             # these decorators are known to return the same function
             # object, we may ignore them
             assert '\n' in source
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1,5 +1,6 @@
 from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictObject
 from pypy.objspace.std.mapdict import *
+from pypy.tool.pytest.appsupport import app_hypothesis_given
 
 class Config:
     class objspace:
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -11,6 +11,21 @@
 
 # ____________________________________________________________
 
+def app_hypothesis_given(arg1):
+    from hypothesis import given
+    def decorator(func):
+        @given(arg1)
+        def inner(space, original, arg1):
+            return original(space, space.wrap(arg1))
+
+        @given(arg1)
+        def appdirect(arg1):
+            return func(arg1)
+        appdirect.hypothesis_inner = inner
+        appdirect.original_function = func
+        return appdirect
+    return decorator
+
 class AppCode(object):
     def __init__(self, space, pycode):
         self.code = pycode
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -51,10 +51,17 @@
         if self.config.option.runappdirect:
             return target()
         space = gettestobjspace()
-        filename = self._getdynfilename(target)
-        func = app2interp_temp(target, filename=filename)
+        if hasattr(target, 'hypothesis_inner'):
+            filename = self._getdynfilename(target.original_function)
+            original = app2interp_temp(target.original_function, filename=filename)
+            func = target.hypothesis_inner
+            args = (space, original)
+        else:
+            filename = self._getdynfilename(target)
+            args = (space, )
+            func = app2interp_temp(target, filename=filename)
         print "executing", func
-        self.execute_appex(space, func, space)
+        self.execute_appex(space, func, *args)
 
     def repr_failure(self, excinfo):
         if excinfo.errisinstance(AppError):
diff --git a/pypy/tool/pytest/test/test_hypothesis.py b/pypy/tool/pytest/test/test_hypothesis.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/pytest/test/test_hypothesis.py
@@ -0,0 +1,12 @@
+from pypy.tool.pytest.appsupport import app_hypothesis_given
+import hypothesis.strategies as strategies
+
+
+# test for the app-test hypothesis support
+
+ at app_hypothesis_given(strategies.floats(min_value=1.0, max_value=2.0))
+def app_test_floats(f):
+    assert 1.0 <= f <= 2.0
+    assert f == f # not a NaN
+
+

From pypy.commits at gmail.com  Fri Jun 10 08:46:26 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 10 Jun 2016 05:46:26 -0700 (PDT)
Subject: [pypy-commit] pypy hypothesis-apptest: support for more than one
 argument
Message-ID: <575ab6a2.e7c9c20a.9dfba.47a1@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: hypothesis-apptest
Changeset: r85075:4270eecf42e5
Date: 2016-06-09 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/4270eecf42e5/

Log:	support for more than one argument

diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -11,16 +11,19 @@
 
 # ____________________________________________________________
 
-def app_hypothesis_given(arg1):
+def app_hypothesis_given(*args):
     from hypothesis import given
+    from hypothesis import strategies
     def decorator(func):
-        @given(arg1)
-        def inner(space, original, arg1):
-            return original(space, space.wrap(arg1))
+        tuple_stategy = strategies.tuples(*args)
+        @given(tuple_stategy)
+        def inner(space, original, arg_tuple):
+            args_w = [space.wrap(arg) for arg in arg_tuple]
+            return original(space, *args_w)
 
-        @given(arg1)
-        def appdirect(arg1):
-            return func(arg1)
+        @given(tuple_stategy)
+        def appdirect(arg_tuple):
+            return func(*arg_tuple)
         appdirect.hypothesis_inner = inner
         appdirect.original_function = func
         return appdirect
diff --git a/pypy/tool/pytest/test/test_hypothesis.py b/pypy/tool/pytest/test/test_hypothesis.py
--- a/pypy/tool/pytest/test/test_hypothesis.py
+++ b/pypy/tool/pytest/test/test_hypothesis.py
@@ -9,4 +9,11 @@
     assert 1.0 <= f <= 2.0
     assert f == f # not a NaN
 
+ at app_hypothesis_given(strategies.floats(min_value=1.0, max_value=2.0), strategies.floats(min_value=1.0, max_value=3.0))
+def app_test_2_floats(f, g):
+    assert 1.0 <= f <= 2.0
+    assert 1.0 <= g <= 3.0
+    assert f == f # not a NaN
+    assert g == g # not a NaN
 
+

From pypy.commits at gmail.com  Fri Jun 10 08:46:28 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 10 Jun 2016 05:46:28 -0700 (PDT)
Subject: [pypy-commit] pypy hypothesis-apptest: support AppTest methods
 using hypothesis
Message-ID: <575ab6a4.82e01c0a.6d177.787f@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: hypothesis-apptest
Changeset: r85076:1da0d5d67e2f
Date: 2016-06-10 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1da0d5d67e2f/

Log:	support AppTest methods using hypothesis

	(a lot of hackery :-( )

diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -15,7 +15,11 @@
     from hypothesis import given
     from hypothesis import strategies
     def decorator(func):
+        # this is a bit of a mess, because @given does not work on functions
+        # with *args
         tuple_stategy = strategies.tuples(*args)
+
+        # two versions of the function, one for applevel, one appdirect
         @given(tuple_stategy)
         def inner(space, original, arg_tuple):
             args_w = [space.wrap(arg) for arg in arg_tuple]
@@ -24,8 +28,20 @@
         @given(tuple_stategy)
         def appdirect(arg_tuple):
             return func(*arg_tuple)
+
+        # two versions that work on methods
+        @given(tuple_stategy)
+        def inner_method(space, w_instance, original, arg_tuple):
+            args_w = [space.wrap(arg) for arg in arg_tuple]
+            return original(space, w_instance, *args_w)
+
+        @given(tuple_stategy)
+        def appdirect_method(instance, arg_tuple):
+            return func(instance, *arg_tuple)
         appdirect.hypothesis_inner = inner
         appdirect.original_function = func
+        appdirect.hypothesis_method = inner_method
+        appdirect.appdirect_method = appdirect_method
         return appdirect
     return decorator
 
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -107,12 +107,22 @@
     def runtest(self):
         target = self.obj
         if self.config.option.runappdirect:
+            if hasattr(target.im_func, 'appdirect_method'):
+                return target.im_func.appdirect_method(target.im_self)
             return target()
         space = target.im_self.space
-        filename = self._getdynfilename(target)
-        func = app2interp_temp(target.im_func, filename=filename)
+        func = target.im_func
         w_instance = self.parent.w_instance
-        self.execute_appex(space, func, space, w_instance)
+        if hasattr(func, 'hypothesis_inner'):
+            filename = self._getdynfilename(func.original_function)
+            original = app2interp_temp(func.original_function, filename=filename)
+            func = func.hypothesis_method
+            args = (space, w_instance, original)
+        else:
+            filename = self._getdynfilename(target)
+            func = app2interp_temp(func, filename=filename)
+            args = (space, w_instance)
+        self.execute_appex(space, func, *args)
 
 
 class AppClassInstance(py.test.collect.Instance):
diff --git a/pypy/tool/pytest/test/test_hypothesis.py b/pypy/tool/pytest/test/test_hypothesis.py
--- a/pypy/tool/pytest/test/test_hypothesis.py
+++ b/pypy/tool/pytest/test/test_hypothesis.py
@@ -16,4 +16,8 @@
     assert f == f # not a NaN
     assert g == g # not a NaN
 
-
+class AppTest(object):
+    @app_hypothesis_given(strategies.floats(min_value=1.0, max_value=2.0))
+    def test_floats(self, f):
+        assert 1.0 <= f <= 2.0
+        assert f == f # not a NaN

From pypy.commits at gmail.com  Fri Jun 10 08:51:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 10 Jun 2016 05:51:03 -0700 (PDT)
Subject: [pypy-commit] pypy new-jit-log: catchup with default
Message-ID: <575ab7b7.0e9e1c0a.42103.ffff80ae@mx.google.com>

Author: Richard Plangger 
Branch: new-jit-log
Changeset: r85077:413ecb1bd580
Date: 2016-06-10 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/413ecb1bd580/

Log:	catchup with default

diff too long, truncating to 2000 out of 14596 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,7 @@
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,17 +104,20 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas Stührk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -140,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +175,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,11 +210,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -228,7 +230,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -270,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +297,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -6,7 +6,7 @@
 irc_header = "And now for something completely different"
 
 
-def interactive_console(mainmodule=None, quiet=False):
+def interactive_console(mainmodule=None, quiet=False, future_flags=0):
     # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
     # mimics what CPython does in pythonrun.c
     if not hasattr(sys, 'ps1'):
@@ -37,15 +37,17 @@
             raise ImportError
         from pyrepl.simple_interact import run_multiline_interactive_console
     except ImportError:
-        run_simple_interactive_console(mainmodule)
+        run_simple_interactive_console(mainmodule, future_flags=future_flags)
     else:
-        run_multiline_interactive_console(mainmodule)
+        run_multiline_interactive_console(mainmodule, future_flags=future_flags)
 
-def run_simple_interactive_console(mainmodule):
+def run_simple_interactive_console(mainmodule, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat 
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -4,6 +4,9 @@
 subprocess module on Windows.
 """
 
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
 
 # Declare external Win32 functions
 
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -332,8 +332,8 @@
     def from_buffer(self, python_buffer):
         """Return a  that points to the data of the
         given Python object, which must support the buffer interface.
-        Note that this is not meant to be used on the built-in types str,
-        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        Note that this is not meant to be used on the built-in types
+        str or unicode (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
         """
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -465,6 +463,7 @@
             else:
                 def __nonzero__(self):
                     return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -35,8 +35,11 @@
                                "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError("Unsupported type: %r.  Please file a bug "
-                                   "if you think it should be." % (commontype,))
+                raise api.FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
             result, quals = parser.parse_type_and_quals(cdecl)   # recursive
 
         assert isinstance(result, model.BaseTypeByIdentity)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -814,7 +814,7 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                    prnt("  (void)((p->%s) | 0);  /* check that '%s.%s' is "
                          "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
@@ -991,7 +991,7 @@
             prnt('static int %s(unsigned long long *o)' % funcname)
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
-            prnt('  *o = (unsigned long long)((%s) << 0);'
+            prnt('  *o = (unsigned long long)((%s) | 0);'
                  '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
@@ -1250,7 +1250,7 @@
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
         s = ('_cffi_prim_int(sizeof(%s), (\n'
-             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '           ((%s)-1) | 0 /* check that %s is an integer type */\n'
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -43,11 +43,13 @@
         return short
     return text
 
-def run_multiline_interactive_console(mainmodule=None):
+def run_multiline_interactive_console(mainmodule=None, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,4 +1,5 @@
-# Empty
+import os
+pypydir = os.path.realpath(os.path.dirname(__file__))
 
 # XXX Should be empty again, soon.
 # XXX hack for win64:
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -1,4 +1,4 @@
-import py, pytest, sys, os, textwrap
+import py, pytest, sys, textwrap
 from inspect import isclass
 
 # pytest settings
@@ -10,8 +10,6 @@
 #
 option = None
 
-pypydir = os.path.realpath(os.path.dirname(__file__))
-
 def braindead_deindent(self):
     """monkeypatch that wont end up doing stupid in the python tokenizer"""
     text = '\n'.join(self.lines)
@@ -166,13 +164,6 @@
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
 
 class PyPyClassCollector(py.test.collect.Class):
     # All pypy Test classes have a "space" member.
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -70,9 +70,6 @@
 bz2
     libbz2
 
-lzma (PyPy3 only)
-    liblzma
-
 pyexpat
     libexpat1
 
@@ -98,11 +95,16 @@
 tk
     tk-dev
 
+lzma (PyPy3 only)
+    liblzma
+
+To run untranslated tests, you need the Boehm garbage collector libgc.
+
 On Debian, this is the command to install all build-time dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev
+    tk-dev libgc-dev liblzma-dev
 
 For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
 
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -13,17 +13,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,9 +63,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -74,31 +74,34 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Edd Barrett
   Andreas Stührk
-  Edd Barrett
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -110,7 +113,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -126,11 +128,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -141,9 +145,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -153,8 +157,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -178,11 +180,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -198,7 +200,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -240,8 +241,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -265,9 +267,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
    release-5.0.1.rst
@@ -49,6 +50,13 @@
    release-0.6
 
 
+CPython 3.3 compatible versions
+-------------------------------
+
+.. toctree::
+
+   release-pypy3.3-v5.2-alpha1.rst
+
 CPython 3.2 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
    whatsnew-4.0.1.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,15 +53,17 @@
 immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated.
 
 
-Numpy improvements
-------------------
+NumPy rebooted
+--------------
 
-The numpy is rapidly progressing in pypy, so feel free to come to IRC and
-ask for proposed topic. A not necesarilly up-to-date `list of topics`_
-is also available.
+Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
+Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+test suite. We could use help analyzing the failures and fixing them either
+as patches to upstream NumPy, or as fixes to PyPy.
 
-.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt
-
+We also are looking for help in how to hijack NumPy dtype conversion and
+ufunc calls to allow the JIT to make them fast, using our internal _numpypy
+module.
 
 Improving the jitviewer
 ------------------------
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -0,0 +1,193 @@
+============
+PyPy2.7 v5.3
+============
+
+We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after
+`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3
+compatibility. This new PyPy2.7 release includes further improvements for the
+CAPI compatibility layer which we call cpyext. In addtion to complete support
+for lxml, we now pass most (more than 90%) of the upstream numpy test suite,
+and much of SciPy is supported as well.
+
+We updated cffi_ to version 1.7 (small changes, documented here_).
+
+.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html
+.. _cffi: https://cffi.readthedocs.org
+.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html
+
+You can download the PyPy2.7 v5.3 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.1 released in April 2016)
+=========================================================
+
+* New features:
+
+  * Merge a major expansion of the C-API support in cpyext, here are some of
+    the highlights:
+
+      - allow c-snippet tests to be run with -A so we can verify we are compatible
+      - fix many edge cases exposed by fixing tests to run with -A
+      - issequence() logic matches cpython
+      - make PyStringObject and PyUnicodeObject field names compatible with cpython
+      - add prelminary support for PyDateTime_*
+      - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
+        PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
+        PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo
+      - improve support for PyGILState_Ensure, PyGILState_Release, and thread
+        primitives, also find a case where CPython will allow thread creation
+        before PyEval_InitThreads is run, dissallow on PyPy 
+      - create a PyObject-specific list strategy
+      - rewrite slot assignment for typeobjects
+      - improve tracking of PyObject to rpython object mapping
+      - support tp_as_{number, sequence, mapping, buffer} slots
+      - support ByteArrayObject via the new resizable_list_supporting_raw_ptr
+      - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's
+      - fix the signature of PyUFunc_FromFuncAndDataAndSignature
+      - implement many PyWhatever_FOO() as a macro taking a `void *`
+
+  * CPyExt tweak: instead of "GIL not held when a CPython C extension module
+    calls PyXxx", we now silently acquire/release the GIL.  Helps with
+    CPython C extension modules that call some PyXxx() functions without
+    holding the GIL (arguably, they are theorically buggy).
+
+  * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
+    It is a more flexible way to make RPython finalizers. Use this mechanism to
+    clean up handling of ``__del__`` methods, fixing issue #2287
+
+  * Generalize cpyext old-style buffers to more than just str/buffer, add
+    support for mmap
+
+  * Support command line -v to trace import statements
+
+  * Add rposix functions for PyPy3.3 support
+
+  * Give super an __init__ and a simple __new__ for CPython compatibility
+
+  * Revive traceviewer, a tool to use pygame to view traces
+
+* Bug Fixes
+
+  * Fix issue #2277: only special-case two exact lists in zip(), not list
+    subclasses, because an overridden __iter__() should be called (probably)
+
+  * Fix issue #2226: Another tweak in the incremental GC- this should ensure
+    that progress in the major GC occurs quickly enough in all cases.
+
+  * Clarify and refactor documentation on http://doc.pypy.org
+
+  * Use "must be unicode, not %T" in unicodedata TypeErrors.
+
+  * Manually reset sys.settrace() and sys.setprofile() when we're done running.
+    This is not exactly what CPython does, but if we get an exception, unlike
+    CPython, we call functions from the 'traceback' module, and these would
+    call more the trace/profile function.  That's unexpected and can lead
+    to more crashes at this point.
+
+  * Use the appropriate tp_dealloc on a subclass of a builtin type, and call
+    tp_new for a python-sublcass of a C-API type
+
+  * Fix for issue #2285 - rare vmprof segfaults on OS/X
+
+  * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc
+
+  * Fix issue #2311 - grab the `__future__` flags imported in the main script, in
+    `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console
+
+  * Issues reported with our previous release were resolved_ after reports from users on
+    our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+    #pypy
+
+* Numpy_:
+
+  * Implement ufunc.outer on numpypy
+
+  * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_
+    accordingly)
+
+* Performance improvements:
+
+  * Use bitstrings to compress lists of descriptors that are attached to an
+    EffectInfo
+
+  * Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+    quite some code internally, and allows the JIT to do better
+    optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+    can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+    negative.
+
+  * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())`
+
+  * Use the __builtin_add_overflow built-ins if they are available
+
+  * Rework the way registers are moved/spilled in before_call()
+
+* Internal refactorings:
+
+  * Refactor code to better support Python3-compatible syntax
+
+  * Document and refactor OperationError -> oefmt
+
+  * Reduce the size of generated C sources during translation by 
+    eliminating many many unused struct declarations (Issue #2281)
+
+  * Remove a number of translation-time options that were not tested and
+    never used. Also fix a performance bug in the method cache
+
+  * Reduce the size of generated code by using the same function objects in
+    all generated subclasses
+
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
+
+  * Compile c snippets with -Werror, and fix warnings it exposed
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+.. _Numpy: https://bitbucket.org/pypy/numpy
+.. _`the repo`: https://bitbucket.org/pypy/numpy
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
@@ -0,0 +1,69 @@
+===================
+PyPy3 v5.2 alpha 1
+===================
+
+We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the
+first release of PyPy which targets Python 3.3 (3.3.5) compatibility.
+
+We would like to thank all of the people who donated_ to the `py3k proposal`_
+for supporting the work that went into this and future releases.
+
+You can download the PyPy3.3 v5.2 alpha 1 release here:
+
+    http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2
+
+Highlights
+==========
+
+* Python 3.3.5 support!
+
+  - Being an early alpha release, there are some `missing features`_ such as a
+    `PEP 393-like space efficient string representation`_ and `known issues`_
+    including performance regressions (e.g. issue `#2305`_). The focus for this
+    release has been updating to 3.3 compatibility. Windows is also not yet
+    supported.
+
+* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4).
+
+What is PyPy?
+==============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT
+compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems except Windows
+    (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+Please try it out and let us know what you think. We welcome feedback, we know
+you are using PyPy, please tell us about it!
+
+We'd especially like to thank these people for their contributions to this
+release:
+
+Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey,
+Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others.
+
+Cheers
+
+The PyPy Team
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation
+.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement
+.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29
+.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305
+.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip
+.. _`dynamic languages`: http://pypyjs.org
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -73,6 +73,8 @@
     'Richard Lancaster':['richardlancaster'],
     'William Leslie':['William ML Leslie'],
     'Spenser Bauman':['Spenser Andrew Bauman'],
+    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
+    'timo':['timo at eistee.fritz.box'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,107 +1,21 @@
 =========================
-What's new in PyPy 5.1+
+What's new in PyPy2.7 5.3+
 =========================
 
-.. this is a revision shortly after release-5.1
-.. startrev: aa60332382a1
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
 
-.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+.. branch: fix-gen-dfa
 
-.. branch: gcheader-decl
+Resolves an issue with the generator script to build the dfa for Python syntax.
 
-Reduce the size of generated C sources.
+.. branch: z196-support
 
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
 
-.. branch: remove-objspace-options
+.. branch: s390x-5.3-catchup
 
-Remove a number of options from the build process that were never tested and
-never set. Fix a performance bug in the method cache.
-
-.. branch: bitstring
-
-JIT: use bitstrings to compress the lists of read or written descrs
-that we attach to EffectInfo.  Fixes a problem we had in
-remove-objspace-options.
-
-.. branch: cpyext-for-merge
-
-Update cpyext C-API support After this branch, we are almost able to support 
-upstream numpy via cpyext, so we created (yet another) fork of numpy at 
-github.com/pypy/numpy with the needed changes. Among the significant changes 
-to cpyext:
-  - allow c-snippet tests to be run with -A so we can verify we are compatible
-  - fix many edge cases exposed by fixing tests to run with -A
-  - issequence() logic matches cpython
-  - make PyStringObject and PyUnicodeObject field names compatible with cpython
-  - add prelminary support for PyDateTime_*
-  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
-    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-  - PyAnySet_CheckExact, PyUnicode_Concat
-  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
-    primitives, also find a case where CPython will allow thread creation
-    before PyEval_InitThreads is run, dissallow on PyPy 
-  - create a PyObject-specific list strategy
-  - rewrite slot assignment for typeobjects
-  - improve tracking of PyObject to rpython object mapping
-  - support tp_as_{number, sequence, mapping, buffer} slots
-
-(makes the pypy-c bigger; this was fixed subsequently by the
-share-cpyext-cpython-api branch)
-
-.. branch: share-mapdict-methods-2
-
-Reduce generated code for subclasses by using the same function objects in all
-generated subclasses.
-
-.. branch: share-cpyext-cpython-api
-
-.. branch: cpyext-auto-gil
-
-CPyExt tweak: instead of "GIL not held when a CPython C extension module
-calls PyXxx", we now silently acquire/release the GIL.  Helps with
-CPython C extension modules that call some PyXxx() functions without
-holding the GIL (arguably, they are theorically buggy).
-
-.. branch: cpyext-test-A
-
-Get the cpyext tests to pass with "-A" (i.e. when tested directly with
-CPython).
-
-.. branch: oefmt
-
-.. branch: cpyext-werror
-
-Compile c snippets with -Werror in cpyext
-
-.. branch: gc-del-3
-
-Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
-It is a more flexible way to make RPython finalizers.
-
-.. branch: unpacking-cpython-shortcut
-
-.. branch: cleanups
-
-.. branch: cpyext-more-slots
-
-.. branch: use-gc-del-3
-
-Use the new rgc.FinalizerQueue mechanism to clean up the handling of
-``__del__`` methods.  Fixes notably issue #2287.  (All RPython
-subclasses of W_Root need to use FinalizerQueue now.)
-
-.. branch: ufunc-outer
-
-Implement ufunc.outer on numpypy
-
-.. branch: verbose-imports
-
-Support ``pypy -v``: verbose imports.  It does not log as much as
-cpython, but it should be enough to help when debugging package layout
-problems.
-
-.. branch: cpyext-macros-cast
-
-Fix some warnings when compiling CPython C extension modules
-
-.. branch: syntax_fix
+Implement the backend related changes for s390x.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -1,5 +1,5 @@
 =========================
-What's new in PyPy 5.1+
+What's new in PyPy2.7 5.3
 =========================
 
 .. this is a revision shortly after release-5.1
@@ -105,3 +105,41 @@
 Fix some warnings when compiling CPython C extension modules
 
 .. branch: syntax_fix
+
+.. branch: remove-raisingops
+
+Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+quite some code internally, and allows the JIT to do better
+optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+negative.
+
+.. branch: cpyext-old-buffers
+
+Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
+
+.. branch: numpy-includes
+
+Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
+This allows building upstream numpy and scipy in pypy via cpyext
+
+.. branch: traceviewer-common-merge-point-formats
+
+Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
+
+.. branch: cpyext-pickle
+
+Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
+at cpyext import time
+
+.. branch: nonmovable-list
+
+Add a way to ask "give me a raw pointer to this list's
+items".  Only for resizable lists of primitives.  Turns the GcArray
+nonmovable, possibly making a copy of it first.
+
+.. branch: cpyext-ext
+
+Finish the work already partially merged in cpyext-for-merge. Adds support
+for ByteArrayObject using the nonmovable-list, which also enables
+buffer(bytearray()) 
diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
@@ -0,0 +1,10 @@
+=================================
+What's new in PyPy3 5.1.1 alpha 1
+=================================
+
+.. A recent revision, ignoring all other branches for this release
+.. startrev: 29d14733e007
+
+.. branch: py3.3
+
+Python 3.3 compatibility
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -238,6 +238,15 @@
 for use. The release packaging script will pick up the tcltk runtime in the lib
 directory and put it in the archive.
 
+The lzma compression library
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.3 ship with CFFI wrappers for the lzma library, which can be
+downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
+5.0.5, a prebuilt version can be downloaded from
+http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
+http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
+
 
 Using the mingw compiler
 ------------------------
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -9,7 +9,7 @@
 from rpython.config.config import to_optparse, make_dict, SUPPRESS_USAGE
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_objspace
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.rlib import rthread
 from pypy.module.thread import os_thread
 
@@ -293,7 +293,7 @@
             self.hack_for_cffi_modules(driver)
 
         return self.get_entry_point(config)
-    
+
     def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
@@ -320,7 +320,7 @@
             while not basedir.join('include').exists():
                 _basedir = basedir.dirpath()
                 if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo', 
+                    raise ValueError('interpreter %s not inside pypy repo',
                                      str(exename))
                 basedir = _basedir
             modules = self.config.objspace.usemodules.getpaths()
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -583,6 +583,12 @@
         if hasattr(signal, 'SIGXFSZ'):
             signal.signal(signal.SIGXFSZ, signal.SIG_IGN)
 
+    # Pre-load the default encoder (controlled by PYTHONIOENCODING) now.
+    # This is needed before someone mucks up with sys.path (or even adds
+    # a unicode string to it, leading to infinite recursion when we try
+    # to encode it during importing).  Note: very obscure.  Issue #2314.
+    str(u'')
+
     def inspect_requested():
         # We get an interactive prompt in one of the following three cases:
         #
@@ -603,6 +609,11 @@
                 ((inspect or (readenv and real_getenv('PYTHONINSPECT')))
                  and sys.stdin.isatty()))
 
+    try:
+        from _ast import PyCF_ACCEPT_NULL_BYTES
+    except ImportError:
+        PyCF_ACCEPT_NULL_BYTES = 0
+    future_flags = [0]
     success = True
 
     try:
@@ -613,7 +624,9 @@
 
             @hidden_applevel
             def run_it():
-                exec run_command in mainmodule.__dict__
+                co_cmd = compile(run_command, '', 'exec')
+                exec co_cmd in mainmodule.__dict__
+                future_flags[0] = co_cmd.co_flags
             success = run_toplevel(run_it)
         elif run_module:
             # handle the "-m" command
@@ -625,11 +638,6 @@
             # handle the case where no command/filename/module is specified
             # on the command-line.
 
-            try:
-                from _ast import PyCF_ACCEPT_NULL_BYTES
-            except ImportError:
-                PyCF_ACCEPT_NULL_BYTES = 0
-
             # update sys.path *after* loading site.py, in case there is a
             # "site.py" file in the script's directory. Only run this if we're
             # executing the interactive prompt, if we're running a script we
@@ -656,6 +664,7 @@
                                                         'exec',
                                                         PyCF_ACCEPT_NULL_BYTES)
                             exec co_python_startup in mainmodule.__dict__
+                            future_flags[0] = co_python_startup.co_flags
                         mainmodule.__file__ = python_startup
                         run_toplevel(run_it)
                         try:
@@ -673,6 +682,7 @@
                     co_stdin = compile(sys.stdin.read(), '', 'exec',
                                        PyCF_ACCEPT_NULL_BYTES)
                     exec co_stdin in mainmodule.__dict__
+                    future_flags[0] = co_stdin.co_flags
                 mainmodule.__file__ = ''
                 success = run_toplevel(run_it)
         else:
@@ -702,7 +712,20 @@
                     args = (runpy._run_module_as_main, '__main__', False)
                 else:
                     # no.  That's the normal path, "pypy stuff.py".
-                    args = (execfile, filename, mainmodule.__dict__)
+                    # This includes the logic from execfile(), tweaked
+                    # to grab the future_flags at the end.
+                    @hidden_applevel
+                    def run_it():
+                        f = file(filename, 'rU')
+                        try:
+                            source = f.read()
+                        finally:
+                            f.close()
+                        co_main = compile(source.rstrip()+"\n", filename,
+                                          'exec', PyCF_ACCEPT_NULL_BYTES)
+                        exec co_main in mainmodule.__dict__
+                        future_flags[0] = co_main.co_flags
+                    args = (run_it,)
             success = run_toplevel(*args)
 
     except SystemExit as e:
@@ -715,12 +738,21 @@
     # start a prompt if requested
     if inspect_requested():
         try:
+            import __future__
             from _pypy_interact import interactive_console
             pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info)
             irc_topic = pypy_version_info[3] != 'final' or (
                             readenv and os.getenv('PYPY_IRC_TOPIC'))
+            flags = 0
+            for fname in __future__.all_feature_names:
+                feature = getattr(__future__, fname)
+                if future_flags[0] & feature.compiler_flag:
+                    flags |= feature.compiler_flag
+            kwds = {}
+            if flags:
+                kwds['future_flags'] = flags
             success = run_toplevel(interactive_console, mainmodule,
-                                   quiet=not irc_topic)
+                                   quiet=not irc_topic, **kwds)
         except SystemExit as e:
             status = e.code
         else:
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -564,7 +564,6 @@
             self.emit_jump(ops.JUMP_FORWARD, end)
             self.use_next_block(next_except)
         self.emit_op(ops.END_FINALLY)   # this END_FINALLY will always re-raise
-        self.is_dead_code()
         self.use_next_block(otherwise)
         self.visit_sequence(te.orelse)
         self.use_next_block(end)
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -1,8 +1,8 @@
 from pypy.interpreter.astcompiler import ast
 class TestAstToObject:
     def test_types(self, space):
-        assert space.is_true(space.issubtype(
-                ast.get(space).w_Module, ast.get(space).w_mod))
+        assert space.issubtype_w(
+                ast.get(space).w_Module, ast.get(space).w_mod)
                                   
     def test_num(self, space):
         value = space.wrap(42)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -243,6 +243,9 @@
     def unicode_w(self, space):
         self._typed_unwrap_error(space, "unicode")
 
+    def bytearray_list_of_chars_w(self, space):
+        self._typed_unwrap_error(space, "bytearray")
+
     def int_w(self, space, allow_conversion=True):
         # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w
         # raises w_TypeError
@@ -1215,7 +1218,7 @@
 
     def abstract_issubclass_w(self, w_cls1, w_cls2):
         # Equivalent to 'issubclass(cls1, cls2)'.
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def abstract_isinstance_w(self, w_obj, w_cls):
         # Equivalent to 'isinstance(obj, cls)'.
@@ -1237,16 +1240,16 @@
     def exception_is_valid_obj_as_class_w(self, w_obj):
         if not self.isinstance_w(w_obj, self.w_type):
             return False
-        return self.is_true(self.issubtype(w_obj, self.w_BaseException))
+        return self.issubtype_w(w_obj, self.w_BaseException)
 
     def exception_is_valid_class_w(self, w_cls):
-        return self.is_true(self.issubtype(w_cls, self.w_BaseException))
+        return self.issubtype_w(w_cls, self.w_BaseException)
 
     def exception_getclass(self, w_obj):
         return self.type(w_obj)
 
     def exception_issubclass_w(self, w_cls1, w_cls2):
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def new_exception_class(self, *args, **kwargs):
         "NOT_RPYTHON; convenience method to create excceptions in modules"
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -205,7 +205,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    map = {"'" : (singleDFA, states_singleDFA),
+           '"' : (doubleDFA, states_doubleDFA),
            "r" : None,
            "R" : None,
            "u" : None,
@@ -257,25 +265,30 @@
     for uniPrefix in ("", "u", "U", "b", "B", ):
         for rawPrefix in ("", "r", "R"):
             prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
+            map[prefix + "'''"] = (single3DFA, states_single3DFA)
+            map[prefix + '"""'] = (double3DFA, states_double3DFA)
     return map
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append('\n')
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -298,22 +311,28 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap["\""]
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.genpytokenize import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -6,7 +6,7 @@
 import sys, os, re, runpy, subprocess
 from rpython.tool.udir import udir
 from contextlib import contextmanager
-from pypy.conftest import pypydir
+from pypy import pypydir
 from lib_pypy._pypy_interact import irc_header
 
 try:
@@ -76,6 +76,11 @@
     print 'Goodbye2'   # should not be reached
     """)
 
+script_with_future = getscript("""
+    from __future__ import division
+    from __future__ import print_function
+    """)
+
 
 class TestParseCommandLine:
     def check_options(self, options, sys_argv, **expected):
@@ -286,7 +291,7 @@
         child.expect('>>>')   # banner
         if irc_topic:
             assert irc_header in child.before
-        else:    
+        else:
             assert irc_header not in child.before
 
     def test_help(self):
@@ -445,6 +450,31 @@
         finally:
             os.environ['PYTHONSTARTUP'] = old
 
+    def test_future_in_executed_script(self):
+        child = self.spawn(['-i', script_with_future])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_python_startup(self, monkeypatch):
+        monkeypatch.setenv('PYTHONSTARTUP', script_with_future)
+        child = self.spawn([])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_cmd(self):
+        child = self.spawn(['-i', '-c', 'from __future__ import division'])
+        child.expect('>>> ')
+        child.sendline('x=1; x/2; 3/4')
+        child.expect('0.5')
+        child.expect('0.75')
+
+    def test_cmd_co_name(self):
+        child = self.spawn(['-c',
+                    'import sys; print sys._getframe(0).f_code.co_name'])
+        child.expect('')
+
     def test_ignore_python_inspect(self):
         os.environ['PYTHONINSPECT_'] = '1'
         try:
@@ -1044,4 +1074,4 @@
             # assert it did not crash
         finally:
             sys.path[:] = old_sys_path
-    
+
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -48,10 +48,10 @@
             return f.f_code
         assert g() is g.func_code
 
-    def test_f_trace_del(self): 
+    def test_f_trace_del(self):
         import sys
-        f = sys._getframe() 
-        del f.f_trace 
+        f = sys._getframe()
+        del f.f_trace
         assert f.f_trace is None
 
     def test_f_lineno(self):
@@ -116,7 +116,7 @@
         def f():
             assert sys._getframe().f_code.co_name == g()
         def g():
-            return sys._getframe().f_back.f_code.co_name 
+            return sys._getframe().f_back.f_code.co_name
         f()
 
     def test_f_back_virtualref(self):
@@ -233,7 +233,7 @@
     def test_trace_exc(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -298,7 +298,7 @@
     def test_trace_return_exc(self):
         import sys
         l = []
-        def trace(a,b,c): 
+        def trace(a,b,c):
             if b in ('exception', 'return'):
                 l.append((b, c))
             return trace
@@ -444,7 +444,7 @@
     def test_dont_trace_on_reraise(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -466,7 +466,7 @@
     def test_dont_trace_on_raise_with_tb(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -3,7 +3,7 @@
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
-class AppTestSlow:    
+class AppTestSlow:
     spaceconfig = dict(usemodules=['itertools'])
 
     def setup_class(cls):
@@ -64,7 +64,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('read_exc_type'),
                   space.wrap(read_exc_type_gw))
-    
+
 def _detach_helpers(space):
     space.delitem(space.builtin.w_dict,
                   space.wrap('hide_top_frame'))
@@ -92,7 +92,7 @@
         pckl = pickle.dumps(code)
         result = pickle.loads(pckl)
         assert code == result
-    
+
     def test_pickle_global_func(self):
         import new
         mod = new.module('mod')
@@ -109,7 +109,7 @@
             assert func is result
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_not_imported_module(self):
         import new
         mod = new.module('mod')
@@ -119,13 +119,13 @@
         result = pickle.loads(pckl)
         assert mod.__name__ == result.__name__
         assert mod.__dict__ == result.__dict__
-    
+
     def test_pickle_builtin_func(self):
         import pickle
         pckl = pickle.dumps(map)
         result = pickle.loads(pckl)
         assert map is result
-    
+
     def test_pickle_non_top_reachable_func(self):
         def func():
             return 42
@@ -142,7 +142,7 @@
         assert func.func_dict     == result.func_dict
         assert func.func_doc      == result.func_doc
         assert func.func_globals  == result.func_globals
-    
+
     def test_pickle_cell(self):
         def g():
             x = [42]
@@ -171,7 +171,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
         assert type(f1) is type(f2)
@@ -223,7 +223,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
     def test_frame_setstate_crash(self):
@@ -257,21 +257,21 @@
         pckl   = pickle.dumps(mod)
         result = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_moduledict(self):
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
         result   = pickle.loads(pckl)
         assert moddict is result
-    
+
     def test_pickle_bltins_module(self):
         import pickle
         mod  = __builtins__
         pckl     = pickle.dumps(mod)
         result   = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_buffer(self):
         skip("Can't pickle buffer objects on top of CPython either.  "
              "Do we really need it?")
@@ -280,14 +280,14 @@
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_complex(self):
         import pickle
         a = complex(1.23,4.567)
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_method(self):
         class myclass(object):
             def f(self):
@@ -308,7 +308,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_staticmethod(self):
         class myclass(object):
             def f():
@@ -319,7 +319,7 @@
         pckl     = pickle.dumps(method)
         result   = pickle.loads(pckl)
         assert method() == result()
-    
+
     def test_pickle_classmethod(self):
         class myclass(object):
             def f(cls):
@@ -337,7 +337,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_sequenceiter(self):
         '''
         In PyPy there is no distinction here between listiterator and
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -12,7 +12,8 @@
 
 
 class TypeDef(object):
-    def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict):
+    def __init__(self, __name, __base=None, __total_ordering__=None,
+                 __buffer=None, **rawdict):
         "NOT_RPYTHON: initialization-time only"
         self.name = __name
         if __base is None:
@@ -22,6 +23,9 @@
         else:
             bases = [__base]
         self.bases = bases
+        # Used in cpyext to fill tp_as_buffer slots
+        assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+        self.buffer = __buffer
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
         # no __del__: use an RPython _finalize_() method and register_finalizer
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -86,8 +86,8 @@
         'max'           : 'functional.max',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
-        'staticmethod'  : 'descriptor.StaticMethod',
-        'classmethod'   : 'descriptor.ClassMethod',
+        'staticmethod'  : 'pypy.interpreter.function.StaticMethod',
+        'classmethod'   : 'pypy.interpreter.function.ClassMethod',
         'property'      : 'descriptor.W_Property',
 
         'globals'       : 'interp_inspect.globals',
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -76,11 +76,10 @@
             w_pretendtype = space.getattr(w_obj, space.wrap('__class__'))
             if space.is_w(w_pretendtype, space.type(w_obj)):
                 return False     # common case: obj.__class__ is type(obj)
-            if allow_override:
-                w_result = space.issubtype_allow_override(w_pretendtype,
-                                                          w_klass_or_tuple)
-            else:
-                w_result = space.issubtype(w_pretendtype, w_klass_or_tuple)
+            if not allow_override:
+                return space.issubtype_w(w_pretendtype, w_klass_or_tuple)
+            w_result = space.issubtype_allow_override(w_pretendtype,
+                                                      w_klass_or_tuple)
         except OperationError as e:
             if e.async(space):
                 raise
@@ -137,11 +136,9 @@
 
     # -- case (type, type)
     try:
-        if allow_override:
-            w_result = space.issubtype_allow_override(w_derived,
-                                                      w_klass_or_tuple)
-        else:
-            w_result = space.issubtype(w_derived, w_klass_or_tuple)
+        if not allow_override:
+            return space.issubtype_w(w_derived, w_klass_or_tuple)
+        w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple)
     except OperationError as e:   # if one of the args was not a type, ignore it

From pypy.commits at gmail.com  Fri Jun 10 09:45:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Jun 2016 06:45:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge branch 'testing-cleanup'
Message-ID: <575ac466.04251c0a.b4768.fffff2d7@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r85078:18006cfc913c
Date: 2016-06-10 14:44 +0100
http://bitbucket.org/pypy/pypy/changeset/18006cfc913c/

Log:	Merge branch 'testing-cleanup'

	Simplify handling of interp-level tests and make it more forward-
	compatible.

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -76,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -110,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -120,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -153,28 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -458,14 +458,17 @@
         decl = str(decl) + "\n"
         yield self.st, decl, 'x', (1, 2, 3, 4)
 
+    def test_closure_error(self):
         source = """if 1:
         def f(a):
             del a
             def x():
                 a
         """
-        exc = py.test.raises(SyntaxError, self.run, source).value
-        assert exc.msg == "Can't delete variable used in nested scopes: 'a'"
+        with py.test.raises(SyntaxError) as excinfo:
+            self.run(source)
+        msg = excinfo.value.msg
+        assert msg == "Can't delete variable used in nested scopes: 'a'"
 
     def test_try_except_finally(self):
         yield self.simple_test, """
@@ -879,7 +882,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -903,19 +919,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
@@ -1111,7 +1115,7 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
-        
+
     def test_list_comprehension(self):
         source = "def f(): [i for i in l]"
         source2 = "def f(): [i for i in l for j in l]"
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -671,13 +671,11 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket', 'array']}
 
     def setup_method(self, method):
-        w_HOST = space.wrap(self.HOST)
-        self.w_serv = space.appexec([w_socket, w_HOST],
+        w_HOST = self.space.wrap(self.HOST)
+        self.w_serv =self.space.appexec([w_socket, w_HOST],
             '''(_socket, HOST):
             serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
             serv.bind((HOST, 0))
@@ -687,7 +685,7 @@
 
     def teardown_method(self, method):
         if hasattr(self, 'w_serv'):
-            space.appexec([self.w_serv], '(serv): serv.close()')
+            self.space.appexec([self.w_serv], '(serv): serv.close()')
             self.w_serv = None
 
     def test_timeout(self):
@@ -803,8 +801,7 @@
 
 
 class AppTestErrno:
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -3,8 +3,9 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
+    spaceconfig = {'usemodules': ['_vmprof', 'struct']}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1')))
         cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2')))
 
@@ -17,7 +18,7 @@
         import struct, sys, gc
 
         WORD = struct.calcsize('l')
-        
+
         def count(s):
             i = 0
             count = 0
@@ -44,7 +45,7 @@
                 else:
                     raise AssertionError(ord(s[i]))
             return count
-        
+
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -13,14 +13,15 @@
 import sys
 import signal
 
+USEMODULES = ['binascii', 'posix', 'struct', 'time']
+if os.name != 'nt':
+    USEMODULES += ['fcntl']
+else:
+    # On windows, os.popen uses the subprocess module
+    USEMODULES += ['_rawffi', 'thread', 'signal']
+
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'time']
-    if os.name != 'nt':
-        usemodules += ['fcntl']
-    else:
-        # On windows, os.popen uses the subprocess module
-        usemodules += ['_rawffi', 'thread', 'signal']
-    mod.space = gettestobjspace(usemodules=usemodules)
+    mod.space = gettestobjspace(usemodules=USEMODULES)
     mod.path = udir.join('posixtestfile.txt')
     mod.path.write("this is a test")
     mod.path2 = udir.join('test_posix2-')
@@ -49,9 +50,10 @@
 
 
 class AppTestPosix:
+    spaceconfig = {'usemodules': USEMODULES}
 
     def setup_class(cls):
-        cls.space = space
+        space = cls.space
         cls.w_runappdirect = space.wrap(cls.runappdirect)
         cls.w_posix = space.appexec([], GET_POSIX)
         cls.w_path = space.wrap(str(path))
@@ -1145,14 +1147,10 @@
 
 class AppTestEnvironment(object):
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name)
-        cls.w_os = space.appexec([], "(): import os; return os")
         cls.w_path = space.wrap(str(path))
 
     def test_environ(self):
-        posix = self.posix
-        os = self.os
+        import posix
         assert posix.environ['PATH']
         del posix.environ['PATH']
         def fn(): posix.environ['PATH']
@@ -1160,7 +1158,7 @@
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
-            os = self.os
+            import os
             os.unsetenv("XYZABC") #does not raise
             try:
                 os.environ["ABCABC"]
@@ -1178,8 +1176,6 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], GET_POSIX)
         if cls.runappdirect:
             # Can't change encoding
             try:
@@ -1187,8 +1183,8 @@
             except UnicodeEncodeError:
                 py.test.skip("encoding not good enough")
         else:
-            cls.save_fs_encoding = space.sys.filesystemencoding
-            space.sys.filesystemencoding = "utf-8"
+            cls.save_fs_encoding = cls.space.sys.filesystemencoding
+            cls.space.sys.filesystemencoding = "utf-8"
 
     def teardown_class(cls):
         try:
@@ -1198,22 +1194,25 @@
 
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
+        import posix
         try:
-            self.posix.stat(u"ą")
+            posix.stat(u"ą")
         except OSError:
             pass
 
     def test_open_unicode(self):
         # Ensure passing unicode doesn't raise UnicodeEncodeError
+        import posix
         try:
-            self.posix.open(u"ą", self.posix.O_WRONLY)
+            posix.open(u"ą", posix.O_WRONLY)
         except OSError:
             pass
 
     def test_remove_unicode(self):
         # See 2 above ;)
+        import posix
         try:
-            self.posix.remove(u"ą")
+            posix.remove(u"ą")
         except OSError:
             pass
 
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.function import Method
 from pypy.tool.pytest import appsupport
 from pypy.tool.pytest.objspace import gettestobjspace
-from pypy.conftest import PyPyClassCollector
 from inspect import getmro
 
 
@@ -21,13 +20,8 @@
     def __init__(self, excinfo):
         self.excinfo = excinfo
 
-marker = py.test.mark.applevel
 
 class AppTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(AppTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
     def _prunetraceback(self, traceback):
         return traceback
 
@@ -122,7 +116,7 @@
             self.w_instance = space.call_function(w_class)
 
 
-class AppClassCollector(PyPyClassCollector):
+class AppClassCollector(py.test.Class):
     Instance = AppClassInstance
 
     def setup(self):
diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py
deleted file mode 100644
--- a/pypy/tool/pytest/inttest.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Collects and executes interpreter-level tests.
-#
-# Most pypy tests are of this kind.
-
-import py
-import sys
-from pypy.interpreter.error import OperationError
-from pypy.conftest import PyPyClassCollector
-
-
-def check_keyboard_interrupt(e):
-    # we cannot easily convert w_KeyboardInterrupt to KeyboardInterrupt
-    # in general without a space -- here is an approximation
-    try:
-        if e.w_type.name == 'KeyboardInterrupt':
-            tb = sys.exc_info()[2]
-            raise KeyboardInterrupt, KeyboardInterrupt(), tb
-    except AttributeError:
-        pass
-
-
-marker = py.test.mark.interplevel
-
-
-class IntTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(IntTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
-    def runtest(self):
-        try:
-            super(IntTestFunction, self).runtest()
-        except OperationError as e:
-            check_keyboard_interrupt(e)
-            raise
-        except Exception as e:
-            cls = e.__class__
-            while cls is not Exception:
-                if cls.__name__ == 'DistutilsPlatformError':
-                    from distutils.errors import DistutilsPlatformError
-                    if isinstance(e, DistutilsPlatformError):
-                        py.test.skip('%s: %s' % (e.__class__.__name__, e))
-                cls = cls.__bases__[0]
-            raise
-
-
-class IntInstanceCollector(py.test.collect.Instance):
-    Function = IntTestFunction
-
-
-class IntClassCollector(PyPyClassCollector):
-    Instance = IntInstanceCollector
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -27,11 +27,11 @@
     result = testdir.runpytest("--collectonly")
     assert result.ret == 0
     result.stdout.fnmatch_lines([
-        "*IntTestFunction*test_func*",
-        "*IntClassCollector*TestClassInt*",
-        "*IntTestFunction*test_method*",
+        "*Function*test_func*",
+        "*Class*TestClassInt*",
+        "*Function*test_method*",
         "*AppClassCollector*AppTestClass*",
-        "*AppTestMethod*", 
+        "*AppTestMethod*",
     ])
 
 class TestSpaceConfig:
@@ -133,5 +133,5 @@
 
     x = 43
     info = raises(ZeroDivisionError, "x/0")
-    assert info.type is ZeroDivisionError    
-    assert isinstance(info.value, ZeroDivisionError)    
+    assert info.type is ZeroDivisionError
+    assert isinstance(info.value, ZeroDivisionError)
diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -20,11 +20,3 @@
         assert not skipped and not failed
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
-
-    def test_appdirect(self, testdir):
-        sorter = testdir.inline_run(innertest, '-m', 'applevel', '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
-        assert len(passed) == 2
-        print passed
-        assert "app_test_something" in passed[0].nodeid
-        assert "test_method_app" in passed[1].nodeid

From pypy.commits at gmail.com  Fri Jun 10 10:12:35 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Jun 2016 07:12:35 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: hg merge testing-default
Message-ID: <575acad3.82a6c20a.17924.7128@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85079:277f12571941
Date: 2016-06-10 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/277f12571941/

Log:	hg merge testing-default

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -94,6 +94,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -128,9 +142,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -138,11 +149,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -171,28 +178,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -965,7 +965,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -989,19 +1002,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -694,13 +694,11 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket', 'array']}
 
     def setup_method(self, method):
-        w_HOST = space.wrap(self.HOST)
-        self.w_serv = space.appexec([w_HOST],
+        w_HOST = self.space.wrap(self.HOST)
+        self.w_serv = self.space.appexec([w_HOST],
             '''(HOST):
             import _socket
             serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
@@ -711,7 +709,7 @@
 
     def teardown_method(self, method):
         if hasattr(self, 'w_serv'):
-            space.appexec([self.w_serv], '(serv): serv.close()')
+            self.space.appexec([self.w_serv], '(serv): serv.close()')
             self.w_serv = None
 
     def test_timeout(self):
@@ -830,8 +828,7 @@
 
 
 class AppTestErrno:
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -3,8 +3,9 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
+    spaceconfig = {'usemodules': ['_vmprof', 'struct']}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1')))
         cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2')))
 
@@ -17,7 +18,7 @@
         import struct, sys, gc
 
         WORD = struct.calcsize('l')
-        
+
         def count(s):
             i = 0
             count = 0
@@ -44,7 +45,7 @@
                 else:
                     raise AssertionError(s[i])
             return count
-        
+
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -10,15 +10,15 @@
 from rpython.translator.c.test.test_extfunc import need_sparse_files
 from rpython.rlib import rposix
 
+USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time']
+# py3k os.open uses subprocess, requiring the following per platform
+if os.name != 'nt':
+    USEMODULES += ['fcntl', 'select', '_posixsubprocess']
+else:
+    USEMODULES += ['_rawffi', 'thread']
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'signal', 'struct', 'time']
-    # py3k os.open uses subprocess, requiring the following per platform
-    if os.name != 'nt':
-        usemodules += ['fcntl', 'select', '_posixsubprocess']
-    else:
-        usemodules += ['_rawffi', 'thread']
-    mod.space = gettestobjspace(usemodules=usemodules)
+    mod.space = gettestobjspace(usemodules=USEMODULES)
     mod.path = udir.join('posixtestfile.txt')
     mod.path.write("this is a test")
     mod.path2 = udir.join('test_posix2-')
@@ -48,9 +48,10 @@
 
 
 class AppTestPosix:
+    spaceconfig = {'usemodules': USEMODULES}
 
     def setup_class(cls):
-        cls.space = space
+        space = cls.space
         cls.w_runappdirect = space.wrap(cls.runappdirect)
         cls.w_posix = space.appexec([], GET_POSIX)
         cls.w_os = space.appexec([], "(): import os as m ; return m")
@@ -1128,14 +1129,11 @@
 
 class AppTestEnvironment(object):
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name)
-        cls.w_os = space.appexec([], "(): import os; return os")
         cls.w_path = space.wrap(str(path))
 
     def test_environ(self):
-        import sys
-        environ = self.posix.environ
+        import sys, posix
+        environ = posix.environ
         item_type = str if sys.platform.startswith('win') else bytes
         for k, v in environ.items():
             assert type(k) is item_type
@@ -1147,7 +1145,7 @@
 
     @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive')
     def test_environ_nonascii(self):
-        import sys
+        import sys, os
         name, value = 'PYPY_TEST_日本', 'foobar日本'
         if not sys.platform == 'win32':
             fsencoding = sys.getfilesystemencoding()
@@ -1158,7 +1156,6 @@
                     skip("Requires %s.encode(sys.getfilesystemencoding(), "
                          "'surogateescape') to succeed (or win32)" % ascii(s))
 
-        os = self.os
         os.environ[name] = value
         assert os.environ[name] == value
         assert os.getenv(name) == value
@@ -1168,7 +1165,7 @@
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
-            os = self.os
+            import os
             os.unsetenv("XYZABC") #does not raise
             try:
                 os.environ["ABCABC"]
@@ -1186,8 +1183,6 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], GET_POSIX)
         if cls.runappdirect:
             # Can't change encoding
             try:
@@ -1195,8 +1190,8 @@
             except UnicodeEncodeError:
                 py.test.skip("encoding not good enough")
         else:
-            cls.save_fs_encoding = space.sys.filesystemencoding
-            space.sys.filesystemencoding = "utf-8"
+            cls.save_fs_encoding = cls.space.sys.filesystemencoding
+            cls.space.sys.filesystemencoding = "utf-8"
 
     def teardown_class(cls):
         try:
@@ -1206,22 +1201,25 @@
 
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
+        import posix
         try:
-            self.posix.stat("ą")
+            posix.stat(u"ą")
         except OSError:
             pass
 
     def test_open_unicode(self):
         # Ensure passing unicode doesn't raise UnicodeEncodeError
+        import posix
         try:
-            self.posix.open("ą", self.posix.O_WRONLY)
+            posix.open(u"ą", posix.O_WRONLY)
         except OSError:
             pass
 
     def test_remove_unicode(self):
         # See 2 above ;)
+        import posix
         try:
-            self.posix.remove("ą")
+            posix.remove(u"ą")
         except OSError:
             pass
 
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -17,7 +17,6 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 from rpython.tool.udir import udir
 from pypy import pypydir
-from pypy.conftest import PyPyClassCollector
 from inspect import getmro
 
 pypyroot = os.path.dirname(pypydir)
@@ -33,7 +32,6 @@
     def __init__(self, excinfo):
         self.excinfo = excinfo
 
-marker = py.test.mark.applevel
 
 def py3k_repr(value):
     "return the repr() that py3k would give for an object."""
@@ -199,10 +197,6 @@
 
 
 class AppTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(AppTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
     def _prunetraceback(self, traceback):
         return traceback
 
@@ -303,7 +297,7 @@
             self.w_instance = space.call_function(w_class)
 
 
-class AppClassCollector(PyPyClassCollector):
+class AppClassCollector(py.test.Class):
     Instance = AppClassInstance
 
     def setup(self):
diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py
deleted file mode 100644
--- a/pypy/tool/pytest/inttest.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Collects and executes interpreter-level tests.
-#
-# Most pypy tests are of this kind.
-
-import py
-import sys
-from pypy.interpreter.error import OperationError
-from pypy.conftest import PyPyClassCollector
-
-
-def check_keyboard_interrupt(e):
-    # we cannot easily convert w_KeyboardInterrupt to KeyboardInterrupt
-    # in general without a space -- here is an approximation
-    try:
-        if e.w_type.name == 'KeyboardInterrupt':
-            tb = sys.exc_info()[2]
-            raise KeyboardInterrupt, KeyboardInterrupt(), tb
-    except AttributeError:
-        pass
-
-
-marker = py.test.mark.interplevel
-
-
-class IntTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(IntTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
-    def runtest(self):
-        try:
-            super(IntTestFunction, self).runtest()
-        except OperationError as e:
-            check_keyboard_interrupt(e)
-            raise
-        except Exception as e:
-            cls = e.__class__
-            while cls is not Exception:
-                if cls.__name__ == 'DistutilsPlatformError':
-                    from distutils.errors import DistutilsPlatformError
-                    if isinstance(e, DistutilsPlatformError):
-                        py.test.skip('%s: %s' % (e.__class__.__name__, e))
-                cls = cls.__bases__[0]
-            raise
-
-
-class IntInstanceCollector(py.test.collect.Instance):
-    Function = IntTestFunction
-
-
-class IntClassCollector(PyPyClassCollector):
-    Instance = IntInstanceCollector
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -27,11 +27,11 @@
     result = testdir.runpytest("--collectonly")
     assert result.ret == 0
     result.stdout.fnmatch_lines([
-        "*IntTestFunction*test_func*",
-        "*IntClassCollector*TestClassInt*",
-        "*IntTestFunction*test_method*",
+        "*Function*test_func*",
+        "*Class*TestClassInt*",
+        "*Function*test_method*",
         "*AppClassCollector*AppTestClass*",
-        "*AppTestMethod*", 
+        "*AppTestMethod*",
     ])
 
 class TestSpaceConfig:
@@ -133,5 +133,5 @@
 
     x = 43
     info = raises(ZeroDivisionError, "x/0")
-    assert info.type is ZeroDivisionError    
-    assert isinstance(info.value, ZeroDivisionError)    
+    assert info.type is ZeroDivisionError
+    assert isinstance(info.value, ZeroDivisionError)
diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -22,15 +22,6 @@
         assert len(failed) == 2
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
-
-    def test_runappdirect(self, testdir):
-        sorter = testdir.inline_run(innertest, '-m', 'applevel -docstring',
-                                    '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
-        assert len(passed) == 4
-        print passed
-        assert "app_test_something" in passed[0].nodeid
-        assert "test_method_app" in passed[1].nodeid
         
     def test_docstring_in_methods(self, testdir): 
         sorter = testdir.inline_run("-k", "AppTestSomething and test_code_in_docstring",

From pypy.commits at gmail.com  Fri Jun 10 12:31:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Jun 2016 09:31:33 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: Blindly try to get tests
 to pass on bencher4
Message-ID: <575aeb65.2450c20a.6ab27.ffffa738@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85080:dc234c6c4b34
Date: 2016-06-10 17:30 +0100
http://bitbucket.org/pypy/pypy/changeset/dc234c6c4b34/

Log:	Blindly try to get tests to pass on bencher4

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1183,6 +1183,7 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
+        cls.w_posix = cls.space.appexec([], GET_POSIX)
         if cls.runappdirect:
             # Can't change encoding
             try:
@@ -1201,25 +1202,22 @@
 
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
-        import posix
         try:
-            posix.stat(u"ą")
+            self.posix.stat(u"ą")
         except OSError:
             pass
 
     def test_open_unicode(self):
         # Ensure passing unicode doesn't raise UnicodeEncodeError
-        import posix
         try:
-            posix.open(u"ą", posix.O_WRONLY)
+            self.posix.open(u"ą", self.posix.O_WRONLY)
         except OSError:
             pass
 
     def test_remove_unicode(self):
         # See 2 above ;)
-        import posix
         try:
-            posix.remove(u"ą")
+            self.posix.remove(u"ą")
         except OSError:
             pass
 

From pypy.commits at gmail.com  Fri Jun 10 13:35:46 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Jun 2016 10:35:46 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: hg merge default
Message-ID: <575afa72.03c31c0a.c2fa1.0d50@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85081:80829afb3cac
Date: 2016-06-10 15:18 +0100
http://bitbucket.org/pypy/pypy/changeset/80829afb3cac/

Log:	hg merge default

diff too long, truncating to 2000 out of 2410 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -176,8 +176,8 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
- * Share cpyext Py* function wrappers according to the signature, shrining the
-   translated libpypy.so by about 
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
 
   * Compile c snippets with -Werror, and fix warnings it exposed
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,3 +5,20 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -205,7 +205,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    map = {"'" : (singleDFA, states_singleDFA),
+           '"' : (doubleDFA, states_doubleDFA),
            "r" : None,
            "R" : None,
            "u" : None,
@@ -257,25 +265,30 @@
     for uniPrefix in ("", "u", "U", "b", "B", ):
         for rawPrefix in ("", "r", "R"):
             prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
+            map[prefix + "'''"] = (single3DFA, states_single3DFA)
+            map[prefix + '"""'] = (double3DFA, states_double3DFA)
     return map
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append('\n')
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -298,22 +311,28 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap["\""]
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.genpytokenize import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -201,6 +201,9 @@
         else:
             copy_string_to_raw(llstr(src_string), dest_data, 0, n)
     else:
+        # nowadays this case should be rare or impossible: as far as
+        # I know, all common types implementing the *writable* buffer
+        # interface now support get_raw_address()
         if src_is_ptr:
             for i in range(n):
                 dest_buf.setitem(i, src_data[i])
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -23,7 +23,7 @@
             guard_true(i14, descr=...)
             guard_not_invalidated(descr=...)
             i16 = int_eq(i6, %d)
-            i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=)
+            i19 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i6, i10, descr=)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
             i22 = int_ge(i19, i10)
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -20,6 +20,10 @@
         self.w_sockets = self.space.wrap([])
         if platform.machine().startswith('arm'):
             self.w_timeout = self.space.wrap(0.06)
+        if platform.machine().startswith('s390x'):
+            # s390x is not slow, but it seems there is one case when epoll
+            # modify method is called that takes longer on s390x
+            self.w_timeout = self.space.wrap(0.06)
         else:
             self.w_timeout = self.space.wrap(0.02)
 
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -3,10 +3,12 @@
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--options] pypy-VER-PLATFORM
+    package.py [--options] --archive-name=pypy-VER-PLATFORM
 
 The output is found in the directory from --builddir,
 by default /tmp/usession-YOURNAME/build/.
+
+For a list of all options, see 'package.py --help'.
 """
 
 import shutil
@@ -61,6 +63,7 @@
     name = options.name
     if not name:
         name = 'pypy-nightly'
+    assert '/' not in name
     rename_pypy_c = options.pypy_c
     override_pypy_c = options.override_pypy_c
 
@@ -288,26 +291,12 @@
         help='destination dir for archive')
     parser.add_argument('--override_pypy_c', type=str, default='',
         help='use as pypy exe instead of pypy/goal/pypy-c')
-    # Positional arguments, for backward compatability with buldbots
-    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
-        metavar='[archive-name] [rename_pypy_c] [targetdir] [override_pypy_c]',
-        )
     options = parser.parse_args(args)
 
-    # Handle positional arguments, choke if both methods are used
-    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
-                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
-        if len(options.extra_args)>i:
-            if getattr(options, target) != default:
-                print 'positional argument',i,target,'already has value',getattr(options, target)
-                parser.print_help()
-                return
-            setattr(options, target, options.extra_args[i])
     if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
         options.nostrip = True
-
     if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        options.tk = True
+        options.no_tk = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -3,7 +3,7 @@
 min=3
 rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-$maj.$min.$rev  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
@@ -34,17 +34,19 @@
 plat=win32
 wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip
 unzip pypy-c-jit-latest-$plat.zip
+rm pypy-c-jit-latest-$plat.zip
 mv pypy-c-jit-*-$plat $rel-$plat
-zip -r $rel-$plat.zip $rel-$plat
+zip -rq $rel-$plat.zip $rel-$plat
 rm -rf $rel-$plat
 
 # Do this after creating a tag, note the untarred directory is pypy-pypy-
 # so make sure there is not another one
 wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
 tar -xf $tagname.tar.bz2
+rm $tagname.tar.bz2
 mv pypy-pypy-* $rel-src
 tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src
-zip -r $rel-src.zip $rel-src
+zip -rq $rel-src.zip $rel-src
 rm -rf $rel-src
 
 # Print out the md5, sha1, sha256
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -21,8 +21,10 @@
 
     def test_dir_structure(self, test='test'):
         retval, builddir = package.package(
-            '--without-cffi', str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--without-cffi',
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
         assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d.%d' % CPYTHON_VERSION[:2]
@@ -71,8 +73,9 @@
         builddir = udir.ensure("build", dir=True)
         retval, builddir = package.package(
             '--without-cffi', '--builddir', str(builddir),
-            str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
 
     def test_with_zipfile_module(self):
         prev = package.USE_ZIPFILE_MODULE
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4610,6 +4610,19 @@
         a.build_types(fd, [])
         py.test.raises(AnnotatorError, a.build_types, fb, [])
 
+    def test_annotate_generator_with_unreachable_yields(self):
+        def f(n):
+            if n < 0:
+                yield 42
+            yield n
+            yield n
+        def main(n):
+            for x in f(abs(n)):
+                pass
+        #
+        a = self.RPythonAnnotator()
+        a.build_types(main, [int])
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/doc/arch/index.rst b/rpython/doc/arch/index.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/arch/index.rst
@@ -0,0 +1,11 @@
+.. _arch_index:
+
+Architecture specific notes
+===========================
+
+Here you can find some architecture specific notes.
+
+.. toctree::
+    :maxdepth: 1
+
+    s390x
diff --git a/rpython/doc/arch/s390x.rst b/rpython/doc/arch/s390x.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/arch/s390x.rst
@@ -0,0 +1,34 @@
+.. _s390x:
+
+IBM Mainframe S390X
+===================
+
+Our JIT implements the 64 bit version of the IBM Mainframe called s390x.
+Note that this architecture is big endian.
+
+Currently supported ISAs:
+
+* z13 (released January 2015)
+* zEC12 (released September 2012)
+* z196 (released August 2010)
+* z10 (released February 2008)
+
+To check if all the necessary CPU facilities are installed
+on the subject machine, please run the test using a copy of the pypy
+source code::
+
+    $ ./pytest.py rpython/jit/backend/zarch/test/test_assembler -v -k 'test_facility'
+
+In addition you can run the auto encoding test to check if your Linux GCC tool chain
+is able to compile all instructions used in the JIT backend::
+
+    $ ./pytest.py rpython/jit/backend/zarch/test/test_auto_encoding.py -v
+
+Translating
+-----------
+
+Specifically check for these two dependencies. On old versions of some
+Linux distributions ship older versions.
+
+* libffi (version should do > 3.0.+).
+* CPython 2.7.+.
diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -37,7 +37,6 @@
 
    arm
    logging
-   s390x
 
 
 Writing your own interpreter in RPython
@@ -61,6 +60,7 @@
    getting-started
    dir-reference
    jit/index
+   arch/index
    translation
    rtyper
    garbage_collection
diff --git a/rpython/doc/s390x.rst b/rpython/doc/s390x.rst
deleted file mode 100644
--- a/rpython/doc/s390x.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _s390x:
-
-S390X JIT Backend
-=================
-
-Our JIT implements the 64 bit version of the IBM Mainframe called s390x.
-Note that this architecture is big endian.
-
-The following facilities need to be installed to operate
-correctly (all of the machines used for development these where installed):
-
-* General-Instructions-Extension
-* Long-Displacement
-* Binary Floating Point (IEEE)
-
-Translating
------------
-
-Ensure that libffi is installed (version should do > 3.0.+).
-CPython should be version 2.7.+.
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -132,13 +132,14 @@
                 del block.operations[index]
                 newlink = split_block(block, index)
                 newblock = newlink.target
+                varnames = get_variable_names(newlink.args)
                 #
                 class Resume(AbstractPosition):
                     _immutable_ = True
+                    _attrs_ = varnames
                     block = newblock
                 Resume.__name__ = 'Resume%d' % len(mappings)
                 mappings.append(Resume)
-                varnames = get_variable_names(newlink.args)
                 #
                 _insert_reads(newblock, varnames)
                 #
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -901,6 +901,8 @@
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
         size = size_box.getint()
+        # hint: try to move unrelated registers away from r0 and r1 now
+        self.rm.spill_or_move_registers_before_call([r.r0, r.r1])
 
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
         t = TempInt()
@@ -924,6 +926,7 @@
         # sizeloc must be in a register, but we can free it now
         # (we take care explicitly of conflicts with r0 or r1)
         sizeloc = self.rm.make_sure_var_in_reg(size_box)
+        self.rm.spill_or_move_registers_before_call([r.r0, r.r1]) # sizeloc safe
         self.rm.possibly_free_var(size_box)
         #
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
@@ -951,6 +954,11 @@
         arraydescr = op.getdescr()
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
+        # can only use spill_or_move_registers_before_call() as a hint if
+        # we are sure that length_box stays alive and won't be freed now
+        # (it should always be the case, see below, but better safe than sorry)
+        if self.rm.stays_alive(length_box):
+            self.rm.spill_or_move_registers_before_call([r.r0, r.r1])
         # the result will be in r0
         self.rm.force_allocate_reg(op, selected_reg=r.r0)
         # we need r1 as a temporary
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -579,11 +579,26 @@
         new_free_regs.append(self.reg_bindings.pop(v))
 
     def before_call(self, force_store=[], save_all_regs=0):
-        """Spill or move some registers before a call.  By default,
-        this means: for every register in 'self.save_around_call_regs',
+        self.spill_or_move_registers_before_call(self.save_around_call_regs,
+                                                 force_store, save_all_regs)
+
+    def spill_or_move_registers_before_call(self, save_sublist,
+                                            force_store=[], save_all_regs=0):
+        """Spill or move some registers before a call.
+
+        By default, this means: for every register in 'save_sublist',
         if there is a variable there and it survives longer than
         the current operation, then it is spilled/moved somewhere else.
 
+        WARNING: this might do the equivalent of possibly_free_vars()
+        on variables dying in the current operation.  It won't
+        immediately overwrite registers that used to be occupied by
+        these variables, though.  Use this function *after* you finished
+        calling self.loc() or self.make_sure_var_in_reg(), i.e. when you
+        know the location of all input arguments.  These locations stay
+        valid, but only *if they are in self.save_around_call_regs,*
+        not if they are callee-saved registers!
+
         'save_all_regs' can be 0 (default set of registers), 1 (do that
         for all registers), or 2 (default + gc ptrs).
 
@@ -612,6 +627,16 @@
         anyway, as a local hack in this function, because on x86 CPUs
         such register-register moves are almost free.
         """
+        if not we_are_translated():
+            # 'save_sublist' is either the whole
+            # 'self.save_around_call_regs', or a sublist thereof, and
+            # then only those registers are spilled/moved.  But when
+            # we move them, we never move them to other registers in
+            # 'self.save_around_call_regs', to avoid ping-pong effects
+            # where the same value is constantly moved around.
+            for reg in save_sublist:
+                assert reg in self.save_around_call_regs
+
         new_free_regs = []
         move_or_spill = []
 
@@ -631,7 +656,7 @@
                 # we need to spill all GC ptrs in this mode
                 self._bc_spill(v, new_free_regs)
                 #
-            elif reg not in self.save_around_call_regs:
+            elif reg not in save_sublist:
                 continue  # in a register like ebx/rbx: it is fine where it is
                 #
             else:
@@ -663,6 +688,7 @@
                 if not we_are_translated():
                     if move_or_spill:
                         assert max_age <= min([_a for _, _a in move_or_spill])
+                    assert reg in save_sublist
                     assert reg in self.save_around_call_regs
                     assert new_reg not in self.save_around_call_regs
                 self.assembler.regalloc_mov(reg, new_reg)
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -324,17 +324,19 @@
         def check(frame):
             expected_size = 1
             idx = 0
+            fixed_size = self.cpu.JITFRAME_FIXED_SIZE
             if self.cpu.backend_name.startswith('arm'):
                 # jitframe fixed part is larger here
                 expected_size = 2
                 idx = 1
+                fixed_size -= 32
             assert len(frame.jf_gcmap) == expected_size
-            if self.cpu.IS_64_BIT:
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE + 1  # +1 from i0
-            else:
-                assert frame.jf_gcmap[idx]
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE - 32 * idx + 1 # +1 from i0
-            assert frame.jf_gcmap[idx] == (1 << (exp_idx + 1)) | (1 << exp_idx)
+            # check that we have two bits set, and that they are in two
+            # registers (p0 and p1 are moved away when doing p2, but not
+            # spilled, just moved to different registers)
+            bits = [n for n in range(fixed_size)
+                      if frame.jf_gcmap[idx] & (1<= self.gc_minimal_size_in_nursery
         constsize = arraydescr.basesize + self.gc_size_of_header
         force_realignment = (itemsize % WORD) != 0
         if force_realignment:
             constsize += WORD - 1
-        self.mc.LEA_ra(edi.value, (eax.value, varsizeloc.value, shift,
+        self.mc.LEA_ra(edx.value, (ecx.value, varsizeloc.value, shift,
                                    constsize))
         if force_realignment:
-            self.mc.AND_ri(edi.value, ~(WORD - 1))
-        # now edi contains the total size in bytes, rounded up to a multiple
+            self.mc.AND_ri(edx.value, ~(WORD - 1))
+        # now edx contains the total size in bytes, rounded up to a multiple
         # of WORD, plus nursery_free_adr
-        self.mc.CMP(edi, heap(nursery_top_adr))
+        self.mc.CMP(edx, heap(nursery_top_adr))
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr1 = self.mc.get_relative_pos()
         #
@@ -2525,8 +2526,8 @@
         self.push_gcmap(self.mc, gcmap, store=True)
         if kind == rewrite.FLAG_ARRAY:
             self.mc.MOV_si(WORD, itemsize)
-            self.mc.MOV(edi, lengthloc)
-            self.mc.MOV_ri(eax.value, arraydescr.tid)
+            self.mc.MOV(edx, lengthloc)
+            self.mc.MOV_ri(ecx.value, arraydescr.tid)
             addr = self.malloc_slowpath_varsize
         else:
             if kind == rewrite.FLAG_STR:
@@ -2534,7 +2535,7 @@
             else:
                 assert kind == rewrite.FLAG_UNICODE
                 addr = self.malloc_slowpath_unicode
-            self.mc.MOV(edi, lengthloc)
+            self.mc.MOV(edx, lengthloc)
         self.mc.CALL(imm(follow_jump(addr)))
         self.mc.JMP_l8(0)      # jump to done, patched later
         jmp_location = self.mc.get_relative_pos()
@@ -2544,9 +2545,9 @@
         self.mc.overwrite(jmp_adr1-1, chr(offset))
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
-        self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
+        self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
-        self.mc.MOV(heap(nursery_free_adr), edi)
+        self.mc.MOV(heap(nursery_free_adr), edx)
         #
         offset = self.mc.get_relative_pos() - jmp_location
         assert 0 < offset <= 127
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -952,14 +952,16 @@
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
         size = size_box.getint()
-        # looking at the result
-        self.rm.force_allocate_reg(op, selected_reg=eax)
+        # hint: try to move unrelated registers away from eax and edx now
+        self.rm.spill_or_move_registers_before_call([ecx, edx])
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
         #
-        # We need edi as a temporary, but otherwise don't save any more
+        # We need edx as a temporary, but otherwise don't save any more
         # register.  See comments in _build_malloc_slowpath().
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
@@ -972,15 +974,16 @@
         size_box = op.getarg(0)
         assert not isinstance(size_box, Const) # we cannot have a const here!
         # sizeloc must be in a register, but we can free it now
-        # (we take care explicitly of conflicts with eax or edi)
+        # (we take care explicitly of conflicts with ecx or edx)
         sizeloc = self.rm.make_sure_var_in_reg(size_box)
+        self.rm.spill_or_move_registers_before_call([ecx, edx])  # sizeloc safe
         self.rm.possibly_free_var(size_box)
-        # the result will be in eax
-        self.rm.force_allocate_reg(op, selected_reg=eax)
-        # we need edi as a temporary
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
+        # we need edx as a temporary
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
@@ -997,16 +1000,21 @@
         arraydescr = op.getdescr()
         length_box = op.getarg(2)
         assert not isinstance(length_box, Const) # we cannot have a const here!
-        # the result will be in eax
-        self.rm.force_allocate_reg(op, selected_reg=eax)
-        # we need edi as a temporary
+        # can only use spill_or_move_registers_before_call() as a hint if
+        # we are sure that length_box stays alive and won't be freed now
+        # (it should always be the case, see below, but better safe than sorry)
+        if self.rm.stays_alive(length_box):
+            self.rm.spill_or_move_registers_before_call([ecx, edx])
+        # the result will be in ecx
+        self.rm.force_allocate_reg(op, selected_reg=ecx)
+        # we need edx as a temporary
         tmp_box = TempVar()
-        self.rm.force_allocate_reg(tmp_box, selected_reg=edi)
-        gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
+        self.rm.force_allocate_reg(tmp_box, selected_reg=edx)
+        gcmap = self.get_gcmap([ecx, edx]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         # length_box always survives: it's typically also present in the
         # next operation that will copy it inside the new array.  It's
-        # fine to load it from the stack too, as long as it's != eax, edi.
+        # fine to load it from the stack too, as long as it is != ecx, edx.
         lengthloc = self.rm.loc(length_box)
         self.rm.possibly_free_var(length_box)
         #
@@ -1225,6 +1233,8 @@
             raise AssertionError("bad unicode item size")
 
     def _consider_math_read_timestamp(self, op):
+        # hint: try to move unrelated registers away from eax and edx now
+        self.rm.spill_or_move_registers_before_call([eax, edx])
         tmpbox_high = TempVar()
         self.rm.force_allocate_reg(tmpbox_high, selected_reg=eax)
         if longlong.is_64_bit:
diff --git a/rpython/jit/backend/x86/test/test_zvmprof.py b/rpython/jit/backend/x86/test/test_zvmprof.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_zvmprof.py
+++ /dev/null
@@ -1,7 +0,0 @@
-
-from rpython.jit.backend.llsupport.test.zrpy_vmprof_test import CompiledVmprofTest
-
-class TestZVMprof(CompiledVmprofTest):
-    
-    gcrootfinder = "shadowstack"
-    gc = "incminimark"
\ No newline at end of file
diff --git a/rpython/jit/backend/zarch/callbuilder.py b/rpython/jit/backend/zarch/callbuilder.py
--- a/rpython/jit/backend/zarch/callbuilder.py
+++ b/rpython/jit/backend/zarch/callbuilder.py
@@ -12,6 +12,8 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.jit.backend.llsupport.descr import CallDescr
 
+CALL_RELEASE_GIL_STACK_OFF = 6*WORD
+
 class CallBuilder(AbstractCallBuilder):
     GPR_ARGS = [r.r2, r.r3, r.r4, r.r5, r.r6]
     FPR_ARGS =  [r.f0, r.f2, r.f4, r.f6]
@@ -85,8 +87,8 @@
         self.subtracted_to_sp += len(stack_params) * WORD
         base = len(stack_params) * WORD
         if self.is_call_release_gil:
-            self.subtracted_to_sp += 8*WORD
-            base += 8*WORD
+            self.subtracted_to_sp += CALL_RELEASE_GIL_STACK_OFF
+            base += CALL_RELEASE_GIL_STACK_OFF
         for idx,i in enumerate(stack_params):
             loc = arglocs[i]
             offset = STD_FRAME_SIZE_IN_BYTES - base + 8 * idx
@@ -187,7 +189,7 @@
         RSHADOWPTR = self.RSHADOWPTR
         RFASTGILPTR = self.RFASTGILPTR
         #
-        pos = STD_FRAME_SIZE_IN_BYTES - 7*WORD
+        pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF
         self.mc.STMG(r.r8, r.r13, l.addr(pos, r.SP))
         #
         # Save this thread's shadowstack pointer into r8, for later comparison
@@ -286,7 +288,7 @@
         if gcrootmap:
             if gcrootmap.is_shadow_stack and self.is_call_release_gil:
                 self.mc.LGR(r.SCRATCH, RSHADOWOLD)
-        pos = STD_FRAME_SIZE_IN_BYTES - 7*WORD
+        pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF
         self.mc.LMG(r.r8, r.r13, l.addr(pos, r.SP))
 
     def write_real_errno(self, save_err):
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -29,6 +29,7 @@
     'MGHI':    ('ri',    ['\xA7','\x0D']),
     'MSGFI':   ('ril',   ['\xC2','\x00']),
     'MLGR':    ('rre',   ['\xB9','\x86'], 'eo,r'),
+    'MLG':     ('rxy',   ['\xE3','\x86'], 'eo,bid'),
     # div/mod
     'DSGR':    ('rre',   ['\xB9','\x0D'], 'eo,r'),
     'DSG':     ('rxy',   ['\xE3','\x0D'], 'eo,bidl'),
@@ -44,7 +45,6 @@
 
     # rotating
     'RISBG':   ('rie_f',   ['\xEC','\x55']),
-    'RISBGN':  ('rie_f',   ['\xEC','\x59']),
 
     # invert & negative & absolute
     'LPGR':    ('rre',   ['\xB9','\x00']),
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -160,11 +160,15 @@
         omc.BRC(c.ANY, l.imm(label_end - jmp_neither_lqlr_overflow))
         omc.overwrite()
 
-    emit_int_floordiv = gen_emit_div_mod('DSGR', 'DSG')
-    emit_uint_floordiv = gen_emit_div_mod('DLGR', 'DLG')
-    # NOTE division sets one register with the modulo value, thus
-    # the regalloc ensures the right register survives.
-    emit_int_mod = gen_emit_div_mod('DSGR', 'DSG')
+    def emit_uint_mul_high(self, op, arglocs, regalloc):
+        r0, _, a1 = arglocs
+        # _ carries the value, contents of r0 are ignored
+        assert not r0.is_imm()
+        assert not a1.is_imm()
+        if a1.is_core_reg():
+            self.mc.MLGR(r0, a1)
+        else:
+            self.mc.MLG(r0, a1)
 
     def emit_int_invert(self, op, arglocs, regalloc):
         l0, = arglocs
diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py
--- a/rpython/jit/backend/zarch/regalloc.py
+++ b/rpython/jit/backend/zarch/regalloc.py
@@ -733,9 +733,6 @@
     prepare_int_sub_ovf = helper.prepare_int_sub
     prepare_int_mul = helper.prepare_int_mul
     prepare_int_mul_ovf = helper.prepare_int_mul_ovf
-    prepare_int_floordiv = helper.prepare_int_div
-    prepare_uint_floordiv = helper.prepare_int_div
-    prepare_int_mod = helper.prepare_int_mod
     prepare_nursery_ptr_increment = prepare_int_add
 
     prepare_int_and = helper.prepare_int_logic
@@ -746,6 +743,18 @@
     prepare_int_lshift  = helper.prepare_int_shift
     prepare_uint_rshift = helper.prepare_int_shift
 
+    def prepare_uint_mul_high(self, op):
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        if a0.is_constant():
+            a0, a1 = a1, a0
+        if helper.check_imm32(a1):
+            l1 = self.ensure_reg(a1)
+        else:
+            l1 = self.ensure_reg_or_pool(a1)
+        lr,lq = self.rm.ensure_even_odd_pair(a0, op, bind_first=True)
+        return [lr, lq, l1]
+
     prepare_int_le = helper.generate_cmp_op()
     prepare_int_lt = helper.generate_cmp_op()
     prepare_int_ge = helper.generate_cmp_op()
diff --git a/rpython/jit/backend/zarch/test/test_assembler.py b/rpython/jit/backend/zarch/test/test_assembler.py
--- a/rpython/jit/backend/zarch/test/test_assembler.py
+++ b/rpython/jit/backend/zarch/test/test_assembler.py
@@ -155,7 +155,15 @@
         s64 = bin(fac_data[1])[2:]
         print(f64)
         print(s64)
+        for i,c in enumerate(f64):
+            print('index: %d is set? %s' % (i,c))
+
+        assert f64[1] == '1' # The z/Architecture architectural mode is installed.
+        assert f64[2] == '1' # The z/Architecture architectural mode is active.
         assert f64[18] == '1' # long displacement facility
+        assert f64[21] == '1' # extended immediate facility
+        assert f64[34] == '1' # general instruction facility
+        assert f64[41] == '1' # floating-point-support-enhancement
 
     def test_load_byte_zero_extend(self):
         adr = self.a.datablockwrapper.malloc_aligned(16, 16)
@@ -189,7 +197,7 @@
     @py.test.mark.parametrize('p', [2**32,2**32+1,2**63-1,2**63-2,0,1,2,3,4,5,6,7,8,10001])
     def test_align_withroll(self, p):
         self.a.mc.load_imm(r.r2, p & 0xffffFFFFffffFFFF)
-        self.a.mc.RISBGN(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0))
+        self.a.mc.RISBG(r.r2, r.r2, loc.imm(0), loc.imm(0x80 | 60), loc.imm(0))
         self.a.mc.BCR(con.ANY, r.r14)
         assert run_asm(self.a) == rffi.cast(rffi.ULONG,p) & ~(7)
 
@@ -214,7 +222,7 @@
         n = 13
         l = loc
         self.a.mc.load_imm(r.r2, 7<> (LONG_BIT - 1)
     return (x ^ mask) - mask
 
+
+def _ll_2_int_floordiv(x, y):
+    # this is used only if the RPython program uses llop.int_floordiv()
+    # explicitly.  For 'a // b', see _handle_int_special() in jtransform.py.
+    # This is the reverse of rpython.rtyper.rint.ll_int_py_div(), i.e.
+    # the same logic as rpython.rtyper.lltypesystem.opimpl.op_int_floordiv
+    # but written in a no-branch style.
+    r = x // y
+    p = r * y
+    # the JIT knows that if x and y are both positive, this is just 'r'
+    return r + (((x ^ y) >> (LONG_BIT - 1)) & (p != x))
+
+def _ll_2_int_mod(x, y):
+    # same comments as _ll_2_int_floordiv()
+    r = x % y
+    # the JIT knows that if x and y are both positive, this doesn't change 'r'
+    r -= y & (((x ^ y) & (r | -r)) >> (LONG_BIT - 1))
+    return r
+
+
 def _ll_1_cast_uint_to_float(x):
     # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
     # (which is a residual call right now in the x86 backend)
@@ -417,6 +437,8 @@
 # in the following calls to builtins, the JIT is allowed to look inside:
 inline_calls_to = [
     ('int_abs',              [lltype.Signed],                lltype.Signed),
+    ('int_floordiv',         [lltype.Signed, lltype.Signed], lltype.Signed),
+    ('int_mod',              [lltype.Signed, lltype.Signed], lltype.Signed),
     ('ll_math.ll_math_sqrt', [lltype.Float],                 lltype.Float),
 ]
 
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -478,7 +478,7 @@
             except ZeroDivisionError:
                 return -42
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_floordiv_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_div_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
@@ -505,7 +505,7 @@
                 return 42
         # XXX so far, this really produces a int_mod_ovf_zer...
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
diff --git a/rpython/jit/codewriter/test/test_support.py b/rpython/jit/codewriter/test/test_support.py
--- a/rpython/jit/codewriter/test/test_support.py
+++ b/rpython/jit/codewriter/test/test_support.py
@@ -3,7 +3,6 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.flowspace.model import Variable, Constant, SpaceOperation
 from rpython.jit.codewriter.support import decode_builtin_call, LLtypeHelpers
-from rpython.jit.codewriter.support import _ll_1_int_abs
 
 def newconst(x):
     return Constant(x, lltype.typeOf(x))
@@ -136,6 +135,7 @@
     py.test.raises(AttributeError, func, llstr(None), p2)
 
 def test_int_abs():
+    from rpython.jit.codewriter.support import _ll_1_int_abs
     assert _ll_1_int_abs(0) == 0
     assert _ll_1_int_abs(1) == 1
     assert _ll_1_int_abs(10) == 10
@@ -143,3 +143,14 @@
     assert _ll_1_int_abs(-1) == 1
     assert _ll_1_int_abs(-10) == 10
     assert _ll_1_int_abs(-sys.maxint) == sys.maxint
+
+def test_int_floordiv_mod():
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.jit.codewriter.support import _ll_2_int_floordiv, _ll_2_int_mod
+    for x in range(-6, 7):
+        for y in range(-3, 4):
+            if y != 0:
+                assert (_ll_2_int_floordiv(x, y) ==
+                        llop.int_floordiv(lltype.Signed, x, y))
+                assert (_ll_2_int_mod(x, y) ==
+                        llop.int_mod(lltype.Signed, x, y))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -97,17 +97,14 @@
         self.emit_operation(op)
 
         r = self.getintbound(op)
-        if b2.is_constant():
-            val = b2.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.is_constant():
-            val = b1.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)):
-            lesser = min(b1.upper, b2.upper)
-            r.intersect(IntBound(0, next_pow2_m1(lesser)))
+        pos1 = b1.known_ge(IntBound(0, 0))
+        pos2 = b2.known_ge(IntBound(0, 0))
+        if pos1 or pos2:
+            r.make_ge(IntBound(0, 0))
+        if pos1:
+            r.make_le(b1)
+        if pos2:
+            r.make_le(b2)
 
     def optimize_INT_SUB(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5188,6 +5188,25 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_and_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        i4 = int_ge(i3, 0)
+        guard_true(i4) []
+        jump(i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        jump(i3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_cmp_above_bounds(self):
         ops = """
         [p0,p1]
@@ -5252,6 +5271,47 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_xor_positive_is_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_lt(i1, 0)
+        guard_false(i3) []
+        i4 = int_xor(i0, i1)
+        i5 = int_lt(i4, 0)
+        guard_false(i5) []
+        jump(i4, i0)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_lt(i1, 0)
+        guard_false(i3) []
+        i4 = int_xor(i0, i1)
+        jump(i4, i0)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_positive_rshift_bits_minus_1(self):
+        ops = """
+        [i0]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        i3 = int_rshift(i2, %d)
+        escape_n(i3)
+        jump(i0)
+        """ % (LONG_BIT - 1,)
+        expected = """
+        [i0]
+        i2 = int_lt(i0, 0)
+        guard_false(i2) []
+        escape_n(0)
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_same_arg(self):
         ops = """
         [i0]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -955,6 +955,75 @@
         res = self.meta_interp(f, [-5])
         assert res == 5+4+3+2+1+0+1+2+3+4+5+6+7+8+9
 
+    def test_int_c_div(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_div(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 // n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_DIV)
+
+    def test_int_c_mod(self):
+        from rpython.rlib.rarithmetic import int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_mod(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_MOD)
+
+    def test_positive_c_div_mod(self):
+        from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                assert i > 0
+                t += int_c_div(100, i) - int_c_mod(100, i)
+                i += 1
+            return t
+        expected = sum([100 // n - 100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # all the correction code should be dead now, xxx test that
+
+    def test_int_c_div_by_constant(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = ['k'], reds = ['i', 't'])
+        def f(i, k):
+            t = 0
+            while i < 100:
+                myjitdriver.can_enter_jit(i=i, t=t, k=k)
+                myjitdriver.jit_merge_point(i=i, t=t, k=k)
+                t += int_c_div(i, k)
+                i += 1
+            return t
+        expected = sum([i // 10 for i in range(51, 100)])
+        assert f(-50, 10) == expected
+        res = self.meta_interp(f, [-50, 10])
+        assert res == expected
+        self.check_resops(call=0, uint_mul_high=2)
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -281,11 +281,12 @@
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
+        "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
-        
+
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
@@ -644,6 +645,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -703,6 +705,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -1139,7 +1142,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj),
+                  "Can't forward an object outside the nursery.")
         tid = self.header(obj).tid
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1463,7 +1467,8 @@
                 objhdr.tid |= GCFLAG_CARDS_SET
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1513,7 +1518,8 @@
             return True
         # ^^^ a fast path of write-barrier
         #
-        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+        if (self.card_page_indices > 0 and     # check constant-folded
+            source_hdr.tid & GCFLAG_HAS_CARDS != 0):
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1548,7 +1554,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1721,12 +1728,15 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(
+            self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "pinned_objects_in_nursery != surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(
+                cur >= prev, "pinned objects encountered in backwards order")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1784,7 +1794,8 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
+                  "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -554,6 +554,7 @@
         assert res # we optimized it
         assert hdr_dst.tid & minimark.GCFLAG_TRACK_YOUNG_PTRS == 0 # and we copied the flag
         #
+        self.gc.card_page_indices = 128     # force > 0
         hdr_src.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_dst.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_src.tid |= minimark.GCFLAG_HAS_CARDS
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -148,7 +148,8 @@
                                                  ('elements', FFI_TYPE_PP)])
 
     ffi_cif = rffi_platform.Struct('ffi_cif', [])
-    ffi_closure = rffi_platform.Struct('ffi_closure', [])
+    ffi_closure = rffi_platform.Struct('ffi_closure',
+                                       [('user_data', rffi.VOIDP)])
 
 def add_simple_type(type_name):
     for name in ['size', 'alignment', 'type']:
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -650,6 +650,26 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_force_ge_zero(lltype.Signed, n)
 
+def int_c_div(x, y):
+    """Return the result of the C-style 'x / y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).  The JIT implements it
+    with a Python-style division followed by correction code.  This
+    is not that bad, because the JIT removes the correction code if
+    x and y are both nonnegative, and if y is any nonnegative constant
+    then the division turns into a rshift or a mul.
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_floordiv(lltype.Signed, x, y)
+
+def int_c_mod(x, y):
+    """Return the result of the C-style 'x % y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_mod(lltype.Signed, x, y)
+
 @objectmodel.specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,17 @@
-#define HAVE_SYS_UCONTEXT_H
+#if !defined(__OpenBSD__)
+#  define HAVE_SYS_UCONTEXT_H
+#else
+#  define HAVE_SIGNAL_H
+#endif
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,10 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(HAVE_SIGNAL_H)
+#include 
+#else
+#  error "don't know how to get the pc on this platform"
 #endif
 
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rlib.rarithmetic import *
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
+from hypothesis import given, strategies
 import sys
 import py
 
@@ -393,6 +394,21 @@
     assert not int_between(1, 2, 2)
     assert not int_between(1, 1, 1)
 
+def test_int_force_ge_zero():
+    assert int_force_ge_zero(42) == 42
+    assert int_force_ge_zero(0) == 0
+    assert int_force_ge_zero(-42) == 0
+
+ at given(strategies.integers(min_value=0, max_value=sys.maxint),
+       strategies.integers(min_value=1, max_value=sys.maxint))
+def test_int_c_div_mod(x, y):
+    assert int_c_div(~x, y) == -(abs(~x) // y)
+    assert int_c_div( x,-y) == -(x // y)
+    assert int_c_div(~x,-y) == +(abs(~x) // y)
+    for x1 in [x, ~x]:
+        for y1 in [y, -y]:
+            assert int_c_div(x1, y1) * y1 + int_c_mod(x1, y1) == x1
+
 # these can't be prebuilt on 32bit
 U1 = r_ulonglong(0x0102030405060708L)
 U2 = r_ulonglong(0x0807060504030201L)
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -236,11 +236,11 @@
         return _rtype_template(hop, 'mul_ovf')
 
     def rtype_floordiv(_, hop):
-        return _rtype_call_helper(hop, 'floordiv', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div', [ZeroDivisionError])
     rtype_inplace_floordiv = rtype_floordiv
 
     def rtype_floordiv_ovf(_, hop):
-        return _rtype_call_helper(hop, 'floordiv_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div_ovf', [ZeroDivisionError])
 
     # turn 'div' on integers into 'floordiv'
     rtype_div         = rtype_floordiv
@@ -250,11 +250,11 @@
     # 'def rtype_truediv' is delegated to the superclass FloatRepr
 
     def rtype_mod(_, hop):
-        return _rtype_call_helper(hop, 'mod', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod', [ZeroDivisionError])
     rtype_inplace_mod = rtype_mod
 
     def rtype_mod_ovf(_, hop):
-        return _rtype_call_helper(hop, 'mod_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod_ovf', [ZeroDivisionError])
 
     def rtype_xor(_, hop):
         return _rtype_template(hop, 'xor')
@@ -319,7 +319,7 @@
     vlist = hop.inputargs(repr, repr2)
     prefix = repr.opprefix
 
-    if '_ovf' in func or func.startswith(('mod', 'floordiv')):
+    if '_ovf' in func or func.startswith(('py_mod', 'py_div')):
         if prefix+func not in ('int_add_ovf', 'int_add_nonneg_ovf',
                                'int_sub_ovf', 'int_mul_ovf'):
             raise TyperError("%r should not be used here any more" % (func,))
@@ -353,7 +353,7 @@
             any_implicit_exception = True
 
     if not any_implicit_exception:
-        if not func.startswith(('mod', 'floordiv')):
+        if not func.startswith(('py_mod', 'py_div')):
             return _rtype_template(hop, func)
 
     repr = hop.r_result
@@ -388,7 +388,7 @@
 # ---------- floordiv ----------
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv(x, y):
+def ll_int_py_div(x, y):
     # Python, and RPython, assume that integer division truncates
     # towards -infinity.  However, in C, integer division truncates
     # towards 0.  So assuming that, we need to apply a correction
@@ -400,159 +400,159 @@
     return r + (u >> INT_BITS_1)
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv_nonnegargs(x, y):
+def ll_int_py_div_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_floordiv(Signed, x, y)            # <= truncates like in C
-    ll_assert(r >= 0, "int_floordiv_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_div_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_floordiv_zer(x, y):

From pypy.commits at gmail.com  Fri Jun 10 13:35:48 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Jun 2016 10:35:48 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Turn the windows error box
 workaround into a fixture
Message-ID: <575afa74.e778c20a.e1927.ffffbaa2@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85082:a8d876855ca5
Date: 2016-06-10 18:34 +0100
http://bitbucket.org/pypy/pypy/changeset/a8d876855ca5/

Log:	Turn the windows error box workaround into a fixture

diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import os
 import pytest
 
 def pytest_configure(config):
@@ -21,3 +21,14 @@
 def pytest_funcarg__api(request):
     return request.cls.api
 
+if os.name == 'nt':
+    @pytest.yield_fixture(autouse=True, scope='session')
+    def prevent_dialog_box():
+        """Do not open dreaded dialog box on segfault on Windows"""
+        import ctypes
+        SEM_NOGPFAULTERRORBOX = 0x0002  # From MSDN
+        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
+        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
+        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
+        yield
+        ctypes.windll.kernel32.SetErrorMode(old_err_mode)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,21 +18,6 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-def setup_module(module):
-    if os.name == 'nt':
-        # Do not open dreaded dialog box on segfault
-        import ctypes
-        SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
-        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
-        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
-        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
-        module.old_err_mode = old_err_mode
-
-def teardown_module(module):
-    if os.name == 'nt':
-        import ctypes
-        ctypes.windll.kernel32.SetErrorMode(module.old_err_mode)
-
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0

From pypy.commits at gmail.com  Fri Jun 10 16:18:25 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 10 Jun 2016 13:18:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: New AST handling in handle_call (not done, backup)
Message-ID: <575b2091.e153c20a.8d0cb.fffff71f@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85083:f4991e923ef5
Date: 2016-06-10 22:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f4991e923ef5/

Log:	New AST handling in handle_call (not done, backup)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -553,7 +553,6 @@
                 break
             if arg_type == tokens.DOUBLESTAR:
                 break
-            #TODO: scan further
             if arg_type == syms.vfpdef or arg_type == syms.tfpdef:
                 n_pos += 1
             if arg_type == tokens.EQUAL:
@@ -1016,11 +1015,11 @@
             return ast.Subscript(left_expr, ast.Index(tup), ast.Load,
                                  middle.get_lineno(), middle.get_column())
 
-    #fix this method
     def handle_call(self, args_node, callable_expr):
-        arg_count = 0
-        keyword_count = 0
-        generator_count = 0
+        arg_count = 0 # position args + iterable args unpackings
+        keyword_count = 0 # keyword args + keyword args unpackings
+        doublestars_count = 0 # just keyword argument unpackings
+        generator_count = 0 
         for i in range(args_node.num_children()):
             argument = args_node.get_child(i)
             if argument.type == syms.argument:
@@ -1028,7 +1027,11 @@
                     arg_count += 1
                 elif argument.get_child(1).type == syms.comp_for:
                     generator_count += 1
+                elif argument.get_child(0).type == tokens.STAR:
+                    arg_count += 1
                 else:
+                    # argument.get_child(0).type == tokens.DOUBLESTAR
+                    # or keyword arg
                     keyword_count += 1
         if generator_count > 1 or \
                 (generator_count and (keyword_count or arg_count)):
@@ -1039,22 +1042,27 @@
         args = []
         keywords = []
         used_keywords = {}
-        variable_arg = None
-        keywords_arg = None
         child_count = args_node.num_children()
         i = 0
         while i < child_count:
             argument = args_node.get_child(i)
             if argument.type == syms.argument:
+                expr_node = argument.get_child(0)
                 if argument.num_children() == 1:
-                    expr_node = argument.get_child(0)
+                    # a positional argument
                     if keywords:
-                        self.error("non-keyword arg after keyword arg",
-                                   expr_node)
-                    if variable_arg:
-                        self.error("only named arguments may follow "
-                                   "*expression", expr_node)
+                        if doublestars_count:
+                            self.error("positional argument follows "
+                                       "keyword argument unpacking",
+                                       expr_node)
+                        else
+                            self.error("positional argument follows "
+                                       "keyword argument",
+                                       expr_node)
                     args.append(self.handle_expr(expr_node))
+                elif expr_node.type == tokens.STAR
+                    # an iterable argument unpacking
+                    # continue here
                 elif argument.get_child(1).type == syms.comp_for:
                     args.append(self.handle_genexp(argument))
                 else:

From pypy.commits at gmail.com  Sat Jun 11 02:38:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 23:38:42 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: 'forward' command
Message-ID: <575bb1f2.4aa71c0a.fbaf5.ffffd25a@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85084:2f0463f67465
Date: 2016-06-11 08:38 +0200
http://bitbucket.org/pypy/pypy/changeset/2f0463f67465/

Log:	'forward' command

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -184,7 +184,8 @@
 static int frozen_pipe_signal[2];
 
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
-static int process_kind = PK_MAIN_PROCESS;
+static unsigned char process_kind = PK_MAIN_PROCESS;
+static unsigned char flag_exit_run_debug_process;
 static uint64_t latest_fork;
 
 static uint64_t total_stop_points;
@@ -316,6 +317,8 @@
         a++;
     }
     if (a->name != NULL) {
+        while (isspace(*p))
+            p++;
         a->act(p);
     }
     else if (strcmp(input, "help") == 0) {
@@ -356,11 +359,25 @@
     return 0;
 }
 
+static void cmd_go(uint64_t target_time)
+{
+    assert(process_kind == PK_DEBUG_PROCESS);
+    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
+               sizeof(target_time));
+    exit(0);
+}
+
 static void check_at_end(uint64_t stop_points)
 {
     char dummy[1];
     uint64_t target_time;
 
+    if (process_kind == PK_DEBUG_PROCESS) {
+        printf("At end.\n");
+        cmd_go(rpy_revdb.stop_point_seen);
+        abort();   /* unreachable */
+    }
+
     if (process_kind != PK_MAIN_PROCESS) {
         fprintf(stderr, "[%d] Unexpectedly falling off the end\n",
                 process_kind);
@@ -502,6 +519,8 @@
             rpy_revdb.stop_point_break = total_stop_points;
         else
             rpy_revdb.stop_point_break += delta;
+        if (rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)
+            rpy_revdb.stop_point_break++;
         close(fds[RD_SIDE]);
         fds[RD_SIDE] = -1;
     }
@@ -509,22 +528,17 @@
 
 static void act_quit(char *p)
 {
-    uint64_t target_time = (uint64_t)-1;
-    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
-               sizeof(target_time));
-    exit(0);
+    cmd_go((uint64_t)-1);
 }
 
 static void act_go(char *p)
 {
-    uint64_t target_time = strtoull(p, NULL, 10);
-    if (target_time == 0) {
-        printf("usage: go \n");
+    int64_t target_time = strtoll(p, NULL, 10);
+    if (target_time <= 0) {
+        printf("usage: go \n");
         return;
     }
-    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
-               sizeof(target_time));
-    exit(0);
+    cmd_go(target_time);
 }
 
 static void act_info_fork(char *p)
@@ -545,16 +559,30 @@
 {
 }
 
+static void act_forward(char *p)
+{
+    int64_t delta = strtoll(p, NULL, 10);
+    if (delta <= 0) {
+        if (delta < 0 || *p == 0)
+            printf("usage: forward \n");
+        return;
+    }
+    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + delta;
+    flag_exit_run_debug_process = 1;
+}
+
 static void run_debug_process(void)
 {
     static struct action_s actions_1[] = {
         { "go", act_go },
+        { "forward", act_forward },
         { "info", act_info },
         { "quit", act_quit },
         { "", act_nop },
         { NULL }
     };
-    while (1) {
+    flag_exit_run_debug_process = 0;
+    while (!flag_exit_run_debug_process) {
         char input[256];
         printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
         fflush(stdout);
@@ -569,12 +597,13 @@
 RPY_EXTERN
 void rpy_reverse_db_break(long stop_point)
 {
-    if (process_kind == PK_MAIN_PROCESS)
+    if (process_kind == PK_MAIN_PROCESS) {
         make_new_frozen_process();
-
-    if (process_kind == PK_DEBUG_PROCESS)
-        if (rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)
-            run_debug_process();
+        if (rpy_revdb.stop_point_seen != rpy_revdb.stop_point_break)
+            return;
+    }
+    assert(process_kind == PK_DEBUG_PROCESS);
+    run_debug_process();
 }
 
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -5,6 +5,11 @@
 from rpython.translator.interactive import Translation
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rlib import revdb
+"""
+These tests require pexpect (UNIX-only).
+http://pexpect.sourceforge.net/
+"""
+import pexpect
 
 
 class RDB(object):
@@ -32,34 +37,39 @@
         return self.cur == len(self.buffer)
 
 
+def compile(self, entry_point, argtypes, backendopt=True):
+    t = Translation(entry_point, None, gc="boehm")
+    self.t = t
+    t.config.translation.reverse_debugger = True
+    t.config.translation.rweakref = False
+    t.config.translation.lldebug0 = True
+    if not backendopt:
+        t.disable(["backendopt_lltype"])
+    t.annotate()
+    t.rtype()
+    if t.backendopt:
+        t.backendopt()
+    self.exename = t.compile_c()
+    self.rdbname = os.path.join(os.path.dirname(str(self.exename)),
+                                'log.rdb')
+
+def run(self, *argv):
+    env = os.environ.copy()
+    env['PYPYREVDB'] = self.rdbname
+    t = self.t
+    stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
+                                               expect_crash=9)
+    print >> sys.stderr, stderr
+    return stdout
+
+def fetch_rdb(self):
+    return RDB(self.rdbname)
+
+
 class BaseTests(object):
-
-    def compile(self, entry_point, argtypes, backendopt=True):
-        t = Translation(entry_point, None, gc="boehm")
-        self.t = t
-        t.config.translation.reverse_debugger = True
-        t.config.translation.rweakref = False
-        if not backendopt:
-            t.disable(["backendopt_lltype"])
-        t.annotate()
-        t.rtype()
-        if t.backendopt:
-            t.backendopt()
-        self.exename = t.compile_c()
-        self.rdbname = os.path.join(os.path.dirname(str(self.exename)),
-                                    'log.rdb')
-
-    def run(self, *argv):
-        env = os.environ.copy()
-        env['PYPYREVDB'] = self.rdbname
-        t = self.t
-        stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
-                                                   expect_crash=9)
-        print >> sys.stderr, stderr
-        return stdout
-
-    def fetch_rdb(self):
-        return RDB(self.rdbname)
+    compile = compile
+    run = run
+    fetch_rdb = fetch_rdb
 
 
 class TestRecording(BaseTests):
@@ -99,50 +109,75 @@
         assert got == [self.exename, 'abc', 'd']
 
 
-class TestInteraction(BaseTests):
-    """
-    These tests require pexpect (UNIX-only).
-    http://pexpect.sourceforge.net/
-    """
+class InteractiveTests(object):
+    EOF = pexpect.EOF
+
     def replay(self, **kwds):
-        import pexpect
-        self.EOF = pexpect.EOF
         kwds.setdefault('timeout', 10)
         child = pexpect.spawn(str(self.exename),
                               ['--replay', str(self.rdbname)], **kwds)
         child.logfile = sys.stdout
+        def expectx(s):
+            child.expect(re.escape(s))
+        assert not hasattr(child, 'expectx')
+        child.expectx = expectx
         return child
 
-    def test_simple_interpreter(self):
+
+class TestSimpleInterpreter(InteractiveTests):
+
+    def setup_class(cls):
         def main(argv):
             for op in argv[1:]:
                 revdb.stop_point(42)
                 print op
             return 9
-        self.compile(main, [], backendopt=False)
-        assert self.run('abc d ef') == 'abc\nd\nef\n'
-        assert self.fetch_rdb().number_of_stop_points() == 3
+        compile(cls, main, [], backendopt=False)
+        assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
+        assert fetch_rdb(cls).number_of_stop_points() == 3
 
+    def test_go(self):
         child = self.replay()
+        child.expectx('stop_points=3\r\n')
+        child.expectx('(3)$ ')
+        child.sendline('go 1')
+        child.expectx('(1)$ ')
+        child.sendline('')
+        child.expectx('(1)$ ')
+        child.sendline('go 52')
+        child.expectx('(3)$ ')
 
-        def wait(s):
-            child.expect(re.escape(s))
+    def test_help(self):
+        child = self.replay()
+        child.sendline('help')
+        child.expectx('select command:\r\n')
+        # ...
+        child.expectx('(3)$ ')
+        child.sendline('info')
+        child.expectx("bad category '', try 'help'\r\n")
 
-        wait('stop_points=3\r\n')
-        wait('(3)$ ')
-        child.sendline('go 1')
-        wait('(1)$ ')
-        child.sendline('')
-        wait('(1)$ ')
-        child.sendline('go 52')
-        wait('(3)$ ')
-        child.sendline('help')
-        wait('select command:\r\n')
-        # ...
-        wait('(3)$ ')
-        child.sendline('info')
-        wait("bad category '', try 'help'\r\n")
+    def test_info_fork(self):
+        child = self.replay()
         child.sendline('info fork')
-        wait('latest_fork=3\r\n')
+        child.expectx('latest_fork=3\r\n')
+
+    def test_quit(self):
+        child = self.replay()
         child.sendline('quit')
         child.expect(self.EOF)
+
+    def test_forward(self):
+        child = self.replay()
+        child.sendline('go 1')
+        child.expectx('(1)$ ')
+        child.sendline('forward 1')
+        child.expectx('(2)$ ')
+        child.sendline('forward 1')
+        child.expectx('(3)$ ')
+        child.sendline('info fork')
+        child.expectx('latest_fork=1\r\n')
+        child.sendline('forward 1')
+        child.expectx('At end.\r\n')
+        child.expectx('(3)$ ')
+        child.sendline('info fork')
+        child.expectx('latest_fork=3\r\n')

From pypy.commits at gmail.com  Sat Jun 11 02:48:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 23:48:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in habnabit/pypy (pull request
 #455)
Message-ID: <575bb426.c68e1c0a.839df.ffffd285@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85090:38b72e0e5401
Date: 2016-06-11 08:47 +0200
http://bitbucket.org/pypy/pypy/changeset/38b72e0e5401/

Log:	Merged in habnabit/pypy (pull request #455)

	Add sys.{get,set}dlopenflags.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1512,7 +1512,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rdynload
 import sys
 
 _WIN = sys.platform == 'win32'
@@ -19,6 +20,7 @@
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
         self.debug = True
+        self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))',
@@ -85,7 +87,9 @@
 
         'float_info'            : 'system.get_float_info(space)',
         'long_info'             : 'system.get_long_info(space)',
-        'float_repr_style'      : 'system.get_float_repr_style(space)'
+        'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'getdlopenflags'        : 'system.getdlopenflags',
+        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -58,3 +58,9 @@
 
 def get_float_repr_style(space):
     return space.wrap("short")
+
+def getdlopenflags(space):
+    return space.wrap(space.sys.dlopenflags)
+
+def setdlopenflags(space, w_flags):
+    space.sys.dlopenflags = space.int_w(w_flags)
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -445,14 +445,12 @@
 
     def test_dlopenflags(self):
         import sys
-        if hasattr(sys, "setdlopenflags"):
-            assert hasattr(sys, "getdlopenflags")
-            raises(TypeError, sys.getdlopenflags, 42)
-            oldflags = sys.getdlopenflags()
-            raises(TypeError, sys.setdlopenflags)
-            sys.setdlopenflags(oldflags+1)
-            assert sys.getdlopenflags() == oldflags+1
-            sys.setdlopenflags(oldflags)
+        raises(TypeError, sys.getdlopenflags, 42)
+        oldflags = sys.getdlopenflags()
+        raises(TypeError, sys.setdlopenflags)
+        sys.setdlopenflags(oldflags+1)
+        assert sys.getdlopenflags() == oldflags+1
+        sys.setdlopenflags(oldflags)
 
     def test_refcount(self):
         import sys
@@ -610,7 +608,7 @@
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
-        
+
         class Tracer:
             def __init__(self):
                 self.events = []
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -26,7 +26,7 @@
 
 if _MAC_OS:
     pre_include_bits = ['#define MACOSX']
-else: 
+else:
     pre_include_bits = []
 
 if _FREEBSD or _NETBSD or _WIN32:
@@ -145,15 +145,20 @@
         else:
             return lltype.nullptr(rffi.VOIDP.TO)
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        mode = RTLD_NOW
+        if RTLD_LOCAL is not None:
+            mode |= RTLD_LOCAL
+        return mode
+
     def dlopen(name, mode=-1):
         """ Wrapper around C-level dlopen
         """
         if mode == -1:
-            if RTLD_LOCAL is not None:
-                mode = RTLD_LOCAL
-            else:
-                mode = 0
-        if (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
+            mode = _dlopen_default_mode()
+        elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
@@ -193,6 +198,11 @@
     DLLHANDLE = rwin32.HMODULE
     RTLD_GLOBAL = None
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        return 0
+
     def dlopen(name, mode=-1):
         # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)

From pypy.commits at gmail.com  Sat Jun 11 02:48:18 2016
From: pypy.commits at gmail.com (Aar...@google.com>)
Date: Fri, 10 Jun 2016 23:48:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add sys.{get,set}dlopenflags.
Message-ID: <575bb432.094ac20a.6bfe0.ffff8313@mx.google.com>

Author: "Aaron Gallagher "
Branch: 
Changeset: r85085:2609c88e9f08
Date: 2016-06-08 15:45 -0700
http://bitbucket.org/pypy/pypy/changeset/2609c88e9f08/

Log:	Add sys.{get,set}dlopenflags.

	Tests already existed for this as part of importing the sys modules
	tests from cpython.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1512,7 +1512,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rdynload
 import sys
 
 _WIN = sys.platform == 'win32'
@@ -19,6 +20,7 @@
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
         self.debug = True
+        self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))',
@@ -85,7 +87,9 @@
 
         'float_info'            : 'system.get_float_info(space)',
         'long_info'             : 'system.get_long_info(space)',
-        'float_repr_style'      : 'system.get_float_repr_style(space)'
+        'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'getdlopenflags'        : 'system.getdlopenflags',
+        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -58,3 +58,9 @@
 
 def get_float_repr_style(space):
     return space.wrap("short")
+
+def getdlopenflags(space):
+    return space.wrap(space.sys.dlopenflags)
+
+def setdlopenflags(space, w_flags):
+    space.sys.dlopenflags = w_flags
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -26,7 +26,7 @@
 
 if _MAC_OS:
     pre_include_bits = ['#define MACOSX']
-else: 
+else:
     pre_include_bits = []
 
 if _FREEBSD or _NETBSD or _WIN32:
@@ -145,14 +145,17 @@
         else:
             return lltype.nullptr(rffi.VOIDP.TO)
 
-    def dlopen(name, mode=-1):
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        mode = RTLD_NOW
+        if RTLD_LOCAL is not None:
+            mode |= RTLD_LOCAL
+        return mode
+
+    def dlopen(name, mode):
         """ Wrapper around C-level dlopen
         """
-        if mode == -1:
-            if RTLD_LOCAL is not None:
-                mode = RTLD_LOCAL
-            else:
-                mode = 0
         if (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
@@ -193,7 +196,12 @@
     DLLHANDLE = rwin32.HMODULE
     RTLD_GLOBAL = None
 
-    def dlopen(name, mode=-1):
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        return 0
+
+    def dlopen(name, mode):
         # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
         if not res:

From pypy.commits at gmail.com  Sat Jun 11 02:48:21 2016
From: pypy.commits at gmail.com (habnabit)
Date: Fri, 10 Jun 2016 23:48:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make it clearer if dlopenflags tests
 ran.
Message-ID: <575bb435.4f941c0a.d84c7.3608@mx.google.com>

Author: Aaron Gallagher 
Branch: 
Changeset: r85087:10f76560e949
Date: 2016-06-08 16:26 -0700
http://bitbucket.org/pypy/pypy/changeset/10f76560e949/

Log:	Make it clearer if dlopenflags tests ran.

	Instead of silently ignoring the test, skip it.

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -445,14 +445,16 @@
 
     def test_dlopenflags(self):
         import sys
-        if hasattr(sys, "setdlopenflags"):
-            assert hasattr(sys, "getdlopenflags")
-            raises(TypeError, sys.getdlopenflags, 42)
-            oldflags = sys.getdlopenflags()
-            raises(TypeError, sys.setdlopenflags)
-            sys.setdlopenflags(oldflags+1)
-            assert sys.getdlopenflags() == oldflags+1
-            sys.setdlopenflags(oldflags)
+        if not hasattr(sys, "setdlopenflags"):
+            skip('dlopen flags are not available.')
+
+        assert hasattr(sys, "getdlopenflags")
+        raises(TypeError, sys.getdlopenflags, 42)
+        oldflags = sys.getdlopenflags()
+        raises(TypeError, sys.setdlopenflags)
+        sys.setdlopenflags(oldflags+1)
+        assert sys.getdlopenflags() == oldflags+1
+        sys.setdlopenflags(oldflags)
 
     def test_refcount(self):
         import sys
@@ -610,7 +612,7 @@
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
-        
+
         class Tracer:
             def __init__(self):
                 self.events = []

From pypy.commits at gmail.com  Sat Jun 11 02:48:23 2016
From: pypy.commits at gmail.com (habnabit)
Date: Fri, 10 Jun 2016 23:48:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Restore mode=-1 default on dlopen.
Message-ID: <575bb437.6a25c20a.614b4.ffffae46@mx.google.com>

Author: Aaron Gallagher 
Branch: 
Changeset: r85088:e2dc439cb4fe
Date: 2016-06-10 15:16 -0700
http://bitbucket.org/pypy/pypy/changeset/e2dc439cb4fe/

Log:	Restore mode=-1 default on dlopen.

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -153,10 +153,12 @@
             mode |= RTLD_LOCAL
         return mode
 
-    def dlopen(name, mode):
+    def dlopen(name, mode=-1):
         """ Wrapper around C-level dlopen
         """
-        if (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
+        if mode == -1:
+            mode = _dlopen_default_mode()
+        elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
@@ -201,7 +203,7 @@
         """
         return 0
 
-    def dlopen(name, mode):
+    def dlopen(name, mode=-1):
         # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
         if not res:

From pypy.commits at gmail.com  Sat Jun 11 02:48:19 2016
From: pypy.commits at gmail.com (habnabit)
Date: Fri, 10 Jun 2016 23:48:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: Unwrap flags before storage.
Message-ID: <575bb433.56311c0a.1104b.ffffd4b3@mx.google.com>

Author: Aaron Gallagher 
Branch: 
Changeset: r85086:d363051e2327
Date: 2016-06-08 16:20 -0700
http://bitbucket.org/pypy/pypy/changeset/d363051e2327/

Log:	Unwrap flags before storage.

	For some reason, the code wasn't failing before, but this will be
	more correct as I understand it.

diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -63,4 +63,4 @@
     return space.wrap(space.sys.dlopenflags)
 
 def setdlopenflags(space, w_flags):
-    space.sys.dlopenflags = w_flags
+    space.sys.dlopenflags = space.int_w(w_flags)

From pypy.commits at gmail.com  Sat Jun 11 02:48:24 2016
From: pypy.commits at gmail.com (habnabit)
Date: Fri, 10 Jun 2016 23:48:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: Don't skip the dlopenflags tests ever.
Message-ID: <575bb438.820b1c0a.5988e.ffffd74f@mx.google.com>

Author: Aaron Gallagher 
Branch: 
Changeset: r85089:f1493c435457
Date: 2016-06-10 15:44 -0700
http://bitbucket.org/pypy/pypy/changeset/f1493c435457/

Log:	Don't skip the dlopenflags tests ever.

	They're not conditionally defined in pypy, so there's no reason to
	skip.

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -445,10 +445,6 @@
 
     def test_dlopenflags(self):
         import sys
-        if not hasattr(sys, "setdlopenflags"):
-            skip('dlopen flags are not available.')
-
-        assert hasattr(sys, "getdlopenflags")
         raises(TypeError, sys.getdlopenflags, 42)
         oldflags = sys.getdlopenflags()
         raises(TypeError, sys.setdlopenflags)

From pypy.commits at gmail.com  Sat Jun 11 02:53:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 23:53:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document branch... er,
 it's not in a branch, but I want to document it anyway
Message-ID: <575bb558.a91cc20a.d7d7b.ffff89ab@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85091:ce97dfe59cde
Date: 2016-06-11 08:51 +0200
http://bitbucket.org/pypy/pypy/changeset/ce97dfe59cde/

Log:	Document branch... er, it's not in a branch, but I want to document
	it anyway

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,9 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.

From pypy.commits at gmail.com  Sat Jun 11 02:53:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Jun 2016 23:53:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: mention branch
Message-ID: <575bb55a.0654c20a.1573.ffff82d5@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85092:19b2d4c0aed3
Date: 2016-06-11 08:53 +0200
http://bitbucket.org/pypy/pypy/changeset/19b2d4c0aed3/

Log:	mention branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,9 @@
 
 .. branch: incminimark-ll_assert
 .. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+

From pypy.commits at gmail.com  Sat Jun 11 04:27:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 01:27:31 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Debug commands written in
 RPython
Message-ID: <575bcb73.073f1c0a.88c73.fffff18e@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85093:9b64e55324a4
Date: 2016-06-11 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/9b64e55324a4/

Log:	Debug commands written in RPython

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1125,6 +1125,13 @@
         exc_data.exc_value = lltype.typeOf(evalue)._defl()
         return bool(etype)
 
+    def op_revdb_stop_point(self, *args):
+        pass
+
+    def op_revdb_send_output(self, ll_string):
+        from rpython.rtyper.annlowlevel import hlstr
+        sys.stdout.write(hlstr(ll_string))
+
 
 class Tracer(object):
     Counter = 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -566,6 +566,7 @@
     'instrument_count':     LLOp(),
 
     'revdb_stop_point':     LLOp(),
+    'revdb_send_output':    LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -159,6 +159,10 @@
 
             self.c_entrypoint_name = pfname
 
+        if self.config.translation.reverse_debugger:
+            from rpython.translator.revdb import revdb_genc
+            revdb_genc.prepare_database(db)
+
         for obj in exports.EXPORTS_obj2name.keys():
             db.getcontainernode(obj)
         exports.clear()
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -7,6 +7,9 @@
 #include 
 #include 
 
+#include "preimpl.h"
+#include "structdef.h"
+#include "src/rtyper.h"
 #include "rdb-src/revdb_include.h"
 
 #define RDB_SIGNATURE   0x0A424452    /* "RDB\n" */
@@ -292,12 +295,36 @@
     return rpy_rev_buffer;
 }
 
+/* generated by RPython */
+extern char *rpy_revdb_command_names[];
+extern void (*rpy_revdb_command_funcs[])(RPyString *);
+
+static void execute_rpy_command(long index, char *arguments)
+{
+    size_t length = strlen(arguments);
+    RPyString *s;
+
+    while (length > 0 && isspace(arguments[length - 1]))
+        length--;
+    s = malloc(sizeof(RPyString) + length);
+    if (s == NULL) {
+        fprintf(stderr, "out of memory\n");
+        exit(1);
+    }
+    /* xxx assumes Boehm here for now */
+    memset(s, 0, sizeof(RPyString));
+    RPyString_Size(s) = length;
+    memcpy(_RPyString_AsString(s), arguments, length);
+
+    rpy_revdb_command_funcs[index](s);
+}
+
 struct action_s {
     const char *name;
     void (*act)(char *);
 };
 
-static void process_input(char *input, const char *kind,
+static void process_input(char *input, const char *kind, int rpycmd,
                           struct action_s actions[])
 {
     char *p;
@@ -310,24 +337,37 @@
         p++;
     if (*p != 0) {
         *p = 0;
-        p++;
+        do {
+            p++;
+        } while (isspace(*p));
     }
-    a = actions;
-    while (a->name != NULL && strcmp(a->name, input) != 0) {
-        a++;
+
+    if (rpycmd) {
+        long i;
+        for (i = 0; rpy_revdb_command_names[i] != NULL; i++) {
+            if (strcmp(rpy_revdb_command_names[i], input) == 0) {
+                execute_rpy_command(i, p);
+                return;
+            }
+        }
     }
-    if (a->name != NULL) {
-        while (isspace(*p))
-            p++;
-        a->act(p);
+
+    for (a = actions; a->name != NULL; a++) {
+        if (strcmp(a->name, input) == 0) {
+            a->act(p);
+            return;
+        }
     }
-    else if (strcmp(input, "help") == 0) {
-        a = actions;
+    if (strcmp(input, "help") == 0) {
         printf("select %s:\n", kind);
-        while (a->name != NULL) {
+        if (rpycmd) {
+            char **p;
+            for (p = rpy_revdb_command_names; *p != NULL; p++)
+                printf("\t%s\n", *p);
+        }
+        for (a = actions; a->name != NULL; a++) {
             if (*a->name != 0)
                 printf("\t%s\n", a->name);
-            a++;
         }
     }
     else {
@@ -552,7 +592,7 @@
         { "fork", act_info_fork },
         { NULL }
     };
-    process_input(p, "category", actions_info);
+    process_input(p, "category", 0, actions_info);
 }
 
 static void act_nop(char *p)
@@ -588,9 +628,10 @@
         fflush(stdout);
         if (fgets(input, sizeof(input), stdin) != input) {
             fprintf(stderr, "\n");
-            strcpy(input, "exit");
+            act_quit("");
+            abort();   /* unreachable */
         }
-        process_input(input, "command", actions_1);
+        process_input(input, "command", 1, actions_1);
     }
 }
 
@@ -606,5 +647,11 @@
     run_debug_process();
 }
 
+RPY_EXTERN
+void rpy_reverse_db_send_output(RPyString *output)
+{
+    fwrite(_RPyString_AsString(output), 1, RPyString_Size(output), stdout);
+}
+
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -56,11 +56,15 @@
 
 #define OP_REVDB_STOP_POINT(stop_point, r)                              \
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
-        rpy_reverse_db_break(stop_point);
+        rpy_reverse_db_break(stop_point)
+
+#define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
+    rpy_reverse_db_send_output(ll_string)
 
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size);
 RPY_EXTERN void rpy_reverse_db_break(long stop_point);
+RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -1,6 +1,7 @@
 import py
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.translator.c.support import cdecl
+from rpython.rlib import exports
 
 
 def extra_files():
@@ -16,3 +17,26 @@
     if tp == 'void @':
         return emit_void(normal_code)
     return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value)
+
+
+def prepare_database(db):
+    FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void))
+
+    bk = db.translator.annotator.bookkeeper
+    cmds = getattr(db.translator, 'revdb_commands', {}).items()
+
+    array_names = lltype.malloc(rffi.CArray(rffi.CCHARP), len(cmds) + 1,
+                                flavor='raw', immortal=True, zero=True)
+    array_funcs = lltype.malloc(rffi.CArray(FUNCPTR), len(cmds),
+                                flavor='raw', immortal=True, zero=True)
+
+    for i, (name, func) in enumerate(cmds):
+        fnptr = lltype.getfunctionptr(bk.getdesc(func).getuniquegraph())
+        assert lltype.typeOf(fnptr) == FUNCPTR
+        array_names[i] = rffi.str2charp(name)
+        array_funcs[i] = fnptr
+
+    exports.EXPORTS_obj2name[array_names._as_obj()] = 'rpy_revdb_command_names'
+    exports.EXPORTS_obj2name[array_funcs._as_obj()] = 'rpy_revdb_command_funcs'
+    db.get(array_names)
+    db.get(array_funcs)
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -181,3 +181,29 @@
         child.expectx('(3)$ ')
         child.sendline('info fork')
         child.expectx('latest_fork=3\r\n')
+
+
+class TestDebugCommands(InteractiveTests):
+
+    def setup_class(cls):
+        #
+        def blip(cmdline):
+            revdb.send_output('<<<' + cmdline + '>>>\n')
+            revdb.send_output('blipped\n')
+        lambda_blip = lambda: blip
+        #
+        def main(argv):
+            revdb.register_debug_command('r', lambda_blip)
+            for op in argv[1:]:
+                revdb.stop_point(42)
+                print op
+            return 9
+        compile(cls, main, [], backendopt=False)
+        assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
+
+    def test_run_blip(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r  foo  bar  baz  ')
+        child.expectx('<<>>\r\nblipped\r\n')
+        child.expectx('(3)$ ')

From pypy.commits at gmail.com  Sat Jun 11 04:47:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 01:47:45 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Can't do I/O in rpython
 commands
Message-ID: <575bd031.41561c0a.4c55f.fffff6b5@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85094:44e6fa91e646
Date: 2016-06-11 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/44e6fa91e646/

Log:	Can't do I/O in rpython commands

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "preimpl.h"
 #include "structdef.h"
@@ -189,6 +190,8 @@
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
 static unsigned char process_kind = PK_MAIN_PROCESS;
 static unsigned char flag_exit_run_debug_process;
+static unsigned char flag_executing_rpython_code;
+static jmp_buf jmp_buf_cancel_execution;
 static uint64_t latest_fork;
 
 static uint64_t total_stop_points;
@@ -284,15 +287,39 @@
 RPY_EXTERN
 char *rpy_reverse_db_fetch(int expected_size)
 {
-    ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p;
-    assert(keep >= 0);
-    memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep);
-    rsize = read_at_least(rpy_rev_buffer + keep,
-                          expected_size - keep,
-                          sizeof(rpy_rev_buffer) - keep);
-    rpy_revdb.buf_p = rpy_rev_buffer;
-    rpy_revdb.buf_limit = rpy_rev_buffer + keep + rsize;
-    return rpy_rev_buffer;
+    if (!flag_executing_rpython_code) {
+        ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p;
+        assert(keep >= 0);
+        memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep);
+        rsize = read_at_least(rpy_rev_buffer + keep,
+                              expected_size - keep,
+                              sizeof(rpy_rev_buffer) - keep);
+        rpy_revdb.buf_p = rpy_rev_buffer;
+        rpy_revdb.buf_limit = rpy_rev_buffer + keep + rsize;
+        return rpy_rev_buffer;
+    }
+    else {
+        /* this is called when we are in execute_rpy_command(): we are
+           running some custom code now, and we can't just perform I/O
+           or access raw memory---because there is no raw memory! 
+        */
+        printf("Attempted to do I/O or access raw memory\n");
+        longjmp(jmp_buf_cancel_execution, 1);
+    }
+}
+
+static void disable_io(rpy_revdb_t *dinfo)
+{
+    *dinfo = rpy_revdb;   /* save the complete struct */
+    rpy_revdb.buf_p = NULL;
+    rpy_revdb.buf_limit = NULL;
+    flag_executing_rpython_code = 1;
+}
+
+static void enable_io(rpy_revdb_t *dinfo)
+{
+    flag_executing_rpython_code = 0;
+    rpy_revdb = *dinfo;
 }
 
 /* generated by RPython */
@@ -303,6 +330,7 @@
 {
     size_t length = strlen(arguments);
     RPyString *s;
+    rpy_revdb_t dinfo;
 
     while (length > 0 && isspace(arguments[length - 1]))
         length--;
@@ -316,7 +344,10 @@
     RPyString_Size(s) = length;
     memcpy(_RPyString_AsString(s), arguments, length);
 
-    rpy_revdb_command_funcs[index](s);
+    disable_io(&dinfo);
+    if (setjmp(jmp_buf_cancel_execution) == 0)
+        rpy_revdb_command_funcs[index](s);
+    enable_io(&dinfo);
 }
 
 struct action_s {
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -189,6 +189,9 @@
         #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
+            if cmdline == 'oops':
+                for i in range(1000):
+                    print 42     # I/O not permitted
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -207,3 +210,22 @@
         child.sendline('r  foo  bar  baz  ')
         child.expectx('<<>>\r\nblipped\r\n')
         child.expectx('(3)$ ')
+
+    def test_io_not_permitted(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r oops')
+        child.expectx('<<>>\r\nAttempted to do I/O or access raw memory')
+        child.expectx('(3)$ ')
+
+    def test_interaction_with_forward(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r oops')
+        child.expectx('<<>>\r\nAttempted to do I/O or access raw memory')
+        child.expectx('(1)$ ')
+        child.sendline('forward 50')
+        child.expectx('At end.\r\n')
+        child.expectx('(3)$ ')

From pypy.commits at gmail.com  Sat Jun 11 05:56:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 02:56:27 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: identityhash()
Message-ID: <575be04b.430ac20a.33013.ffffbf71@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85095:372509ebf296
Date: 2016-06-11 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/372509ebf296/

Log:	identityhash()

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -282,7 +282,8 @@
                "Give an executable that writes a log file for reverse debugging",
                default=False, cmdline='--revdb',
                requires=[('translation.split_gc_address_space', True),
-                         ('translation.jit', False)]),
+                         ('translation.jit', False),
+                         ('translation.gc', 'boehm')]),
 ])
 
 def get_combined_translation_config(other_optdescr=None,
diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -30,13 +30,6 @@
 
         HDRPTR = lltype.Ptr(self.HDR)
 
-        def ll_identityhash(addr):
-            obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
-            h = obj.hash
-            if h == 0:
-                obj.hash = h = ~llmemory.cast_adr_to_int(addr)
-            return h
-
         if self.translator:
             self.malloc_fixedsize_ptr = self.inittime_helper(
                 ll_malloc_fixedsize, [lltype.Signed], llmemory.GCREF)
@@ -52,9 +45,18 @@
                     inline=False)
                 self.weakref_deref_ptr = self.inittime_helper(
                     ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
-            self.identityhash_ptr = self.inittime_helper(
-                ll_identityhash, [llmemory.Address], lltype.Signed,
-                inline=False)
+
+            if not translator.config.translation.reverse_debugger:
+                def ll_identityhash(addr):
+                    obj = llmemory.cast_adr_to_ptr(addr, HDRPTR)
+                    h = obj.hash
+                    if h == 0:
+                        obj.hash = h = ~llmemory.cast_adr_to_int(addr)
+                    return h
+                self.identityhash_ptr = self.inittime_helper(
+                    ll_identityhash, [llmemory.Address], lltype.Signed,
+                    inline=False)
+
             self.mixlevelannotator.finish()   # for now
             self.mixlevelannotator.backend_optimize()
 
@@ -146,10 +148,14 @@
 
     def gct_gc_identityhash(self, hop):
         v_obj = hop.spaceop.args[0]
-        v_adr = hop.genop("cast_ptr_to_adr", [v_obj],
-                          resulttype=llmemory.Address)
-        hop.genop("direct_call", [self.identityhash_ptr, v_adr],
-                  resultvar=hop.spaceop.result)
+        if not self.translator.config.translation.reverse_debugger:
+            v_addr = hop.genop("cast_ptr_to_adr", [v_obj],
+                               resulttype=llmemory.Address)
+            hop.genop("direct_call", [self.identityhash_ptr, v_addr],
+                      resultvar=hop.spaceop.result)
+        else:
+            hop.genop("revdb_identityhash", [v_obj],
+                      resultvar=hop.spaceop.result)
 
     def gct_gc_id(self, hop):
         # this is the logic from the HIDE_POINTER macro in 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -567,6 +567,7 @@
 
     'revdb_stop_point':     LLOp(),
     'revdb_send_output':    LLOp(),
+    'revdb_identityhash':   LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -630,8 +630,8 @@
             gcsrc = isinstance(TSRC, Ptr) and TSRC.TO._gckind == 'gc'
             if gcsrc != gcdst:
                 raise Exception(
-                  "cast between pointer types changes the address space,\n"
-                  "but the 'split_gc_address_space' option is enabled:\n"
+                  "cast between pointer types changes the address\n"
+                  "space, but the 'split_gc_address_space' option is enabled:\n"
                   "  func: %s\n"
                   "    op: %s\n"
                   "  from: %s\n"
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -136,6 +136,24 @@
     write_all(rpy_rev_buffer, size);
 }
 
+RPY_EXTERN
+Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj)
+{
+    /* Boehm only */
+    if (obj->h_hash == 0) {
+        Signed h;
+        /* When recording, we get the hash the normal way from the
+           pointer casted to an int, and record that.  When replaying,
+           we read it from the record.  In both cases, we cache the
+           hash in the object, so that we record/replay only once per
+           object. */
+        RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h);
+        assert(h != 0);
+        obj->h_hash = h;
+    }
+    return obj->h_hash;
+}
+
 
 /* ------------------------------------------------------------ */
 /* Replaying mode                                               */
@@ -190,7 +208,7 @@
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
 static unsigned char process_kind = PK_MAIN_PROCESS;
 static unsigned char flag_exit_run_debug_process;
-static unsigned char flag_executing_rpython_code;
+static unsigned char flag_io_disabled;
 static jmp_buf jmp_buf_cancel_execution;
 static uint64_t latest_fork;
 
@@ -287,7 +305,7 @@
 RPY_EXTERN
 char *rpy_reverse_db_fetch(int expected_size)
 {
-    if (!flag_executing_rpython_code) {
+    if (!flag_io_disabled) {
         ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p;
         assert(keep >= 0);
         memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep);
@@ -313,12 +331,12 @@
     *dinfo = rpy_revdb;   /* save the complete struct */
     rpy_revdb.buf_p = NULL;
     rpy_revdb.buf_limit = NULL;
-    flag_executing_rpython_code = 1;
+    flag_io_disabled = 1;
 }
 
 static void enable_io(rpy_revdb_t *dinfo)
 {
-    flag_executing_rpython_code = 0;
+    flag_io_disabled = 0;
     rpy_revdb = *dinfo;
 }
 
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -61,10 +61,14 @@
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
 
+#define OP_REVDB_IDENTITYHASH(obj, r)                                   \
+    r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
+
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size);
 RPY_EXTERN void rpy_reverse_db_break(long stop_point);
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
+RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -4,7 +4,8 @@
 from rpython.tool.udir import udir
 from rpython.translator.interactive import Translation
 from rpython.rlib.rarithmetic import LONG_BIT
-from rpython.rlib import revdb
+from rpython.rlib import objectmodel, revdb
+from rpython.rlib.rarithmetic import intmask
 """
 These tests require pexpect (UNIX-only).
 http://pexpect.sourceforge.net/
@@ -30,6 +31,24 @@
         self.cur = p + struct.calcsize(mode)
         return struct.unpack_from(mode, self.buffer, p)[0]
 
+    def read_check_argv(self, expected):
+        assert self.argc == len(expected)
+        for i in range(self.argc):
+            self.next()    # this is from "p = argv[i]"
+            s = []
+            # first we determine the length of the "char *p"
+            while True:
+                c = self.next('c')
+                if c == '\x00':
+                    break
+                s.append(c)
+            # then we really read the "char *" and copy it into a rpy string
+            # (that's why this time we don't read the final \0)
+            for c1 in s:
+                c2 = self.next('c')
+                assert c2 == c1
+            assert ''.join(s) == expected[i]
+
     def number_of_stop_points(self):
         return struct.unpack_from("q", self.buffer, len(self.buffer) - 8)[0]
 
@@ -66,14 +85,11 @@
     return RDB(self.rdbname)
 
 
-class BaseTests(object):
+class TestRecording(object):
     compile = compile
     run = run
     fetch_rdb = fetch_rdb
 
-
-class TestRecording(BaseTests):
-
     def test_simple(self):
         def main(argv):
             print argv[1:]
@@ -81,32 +97,35 @@
         self.compile(main, [], backendopt=False)
         assert self.run('abc d') == '[abc, d]\n'
         rdb = self.fetch_rdb()
-        assert rdb.argc == 3
-        #
-        got = []
-        for i in range(3):
-            rdb.next()    # this is from "p = argv[i]"
-            s = []
-            # first we determine the length of the "char *p"
-            while True:
-                c = rdb.next('c')
-                if c == '\x00':
-                    break
-                s.append(c)
-            # then we really read the "char *" and copy it into a rpy string
-            # (that's why this time we don't read the final \0)
-            for c1 in s:
-                c2 = rdb.next('c')
-                assert c2 == c1
-            got.append(''.join(s))
+        rdb.read_check_argv([self.exename, 'abc', 'd'])
         # write() call
         x = rdb.next(); assert x == len('[abc, d]\n')
         x = rdb.next('i'); assert x == 0      # errno
         x = rdb.next('q'); assert x == 0      # number of stop points
         # that's all we should get from this simple example
         assert rdb.done()
-        #
-        assert got == [self.exename, 'abc', 'd']
+
+    def test_identityhash(self):
+        def main(argv):
+            print [objectmodel.compute_identity_hash(argv),
+                   objectmodel.compute_identity_hash(argv),
+                   objectmodel.compute_identity_hash(argv)]
+            return 9
+        self.compile(main, [], backendopt=False)
+        out = self.run('Xx')
+        match = re.match(r'\[(-?\d+), \1, \1]\n', out)
+        assert match
+        hash_value = int(match.group(1))
+        rdb = self.fetch_rdb()
+        rdb.read_check_argv([self.exename, 'Xx'])
+        # compute_identity_hash() call, but only the first one
+        x = rdb.next(); assert intmask(x) == intmask(hash_value)
+        # write() call
+        x = rdb.next(); assert x == len(out)
+        x = rdb.next('i'); assert x == 0      # errno
+        # done
+        x = rdb.next('q'); assert x == 0      # number of stop points
+        assert rdb.done()
 
 
 class InteractiveTests(object):

From pypy.commits at gmail.com  Sat Jun 11 06:32:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 03:32:33 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Tweaks. Can now record and
 replay duhton.
Message-ID: <575be8c1.45271c0a.cb2c7.1deb@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85096:0c5d9de93e74
Date: 2016-06-11 12:33 +0200
http://bitbucket.org/pypy/pypy/changeset/0c5d9de93e74/

Log:	Tweaks. Can now record and replay duhton.

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -102,6 +102,9 @@
             destrptr = None
             DESTR_ARG = None
 
+        if self.translator.config.translation.reverse_debugger:
+            destrptr = None    # XXX for now
+
         if destrptr:
             EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value
             typename = TYPE.__name__
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -5,7 +5,7 @@
 
 import py
 
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -42,6 +42,8 @@
 def stack_check():
     if not we_are_translated():
         return
+    if fetch_translated_config().translation.reverse_debugger:
+        return     # XXX for now
     #
     # Load the "current" stack position, or at least some address that
     # points close to the current stack head
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -95,7 +95,7 @@
 
 static void setup_record_mode(int argc, char *argv[])
 {
-    char *filename = getenv("PYPYREVDB");
+    char *filename = getenv("PYPYRDB");
     rdb_header_t h;
 
     assert(RPY_RDB_REPLAY == 0);
@@ -103,11 +103,11 @@
     rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32;
 
     if (filename && *filename) {
-        putenv("PYPYREVDB=");
+        putenv("PYPYRDB=");
         rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC |
                               O_CREAT | O_NOCTTY | O_TRUNC, 0600);
         if (rpy_rev_fileno < 0) {
-            fprintf(stderr, "Fatal error: can't create PYPYREVDB file '%s'\n",
+            fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n",
                     filename);
             abort();
         }
@@ -473,7 +473,9 @@
         exit(1);
     }
     if (stop_points != rpy_revdb.stop_point_seen) {
-        fprintf(stderr, "Bad number of stop points\n");
+        fprintf(stderr, "Bad number of stop points "
+                "(seen %llu, recorded %llu)\n", rpy_revdb.stop_point_seen,
+                stop_points);
         exit(1);
     }
     if (rpy_revdb.buf_p != rpy_revdb.buf_limit ||
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -28,12 +28,21 @@
 RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]);
 RPY_EXTERN void rpy_reverse_db_teardown(void);
 
+#if 0    /* enable to print locations to stderr of all the EMITs */
+#  define _RPY_REVDB_PRINT(args)  fprintf args
+#else
+#  define _RPY_REVDB_PRINT(args)  /* nothing */
+#endif
+
 
 #define RPY_REVDB_EMIT(normal_code, decl_e, variable)                   \
     if (!RPY_RDB_REPLAY) {                                              \
         normal_code                                                     \
         {                                                               \
             decl_e = variable;                                          \
+            _RPY_REVDB_PRINT((stderr, "%s:%d: write %*llx\n",           \
+                              __FILE__, __LINE__,                       \
+                              2 * sizeof(_e), (unsigned long long)_e)); \
             memcpy(rpy_revdb.buf_p, &_e, sizeof(_e));                   \
             if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit)  \
                 rpy_reverse_db_flush();                                 \
@@ -48,6 +57,9 @@
             }                                                           \
             rpy_revdb.buf_p = _end1;                                    \
             memcpy(&_e, _src, sizeof(_e));                              \
+            _RPY_REVDB_PRINT((stderr, "%s:%d: read %*llx\n",            \
+                              __FILE__, __LINE__,                       \
+                              2 * sizeof(_e), (unsigned long long)_e)); \
             variable = _e;                                              \
     }
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -74,7 +74,7 @@
 
 def run(self, *argv):
     env = os.environ.copy()
-    env['PYPYREVDB'] = self.rdbname
+    env['PYPYRDB'] = self.rdbname
     t = self.t
     stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
                                                expect_crash=9)

From pypy.commits at gmail.com  Sat Jun 11 06:49:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 03:49:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Don't record/replay the reads
 from static, immutable structures (the
Message-ID: <575beccc.49c51c0a.8377a.2061@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85097:59cd9a5ec3f6
Date: 2016-06-11 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/59cd9a5ec3f6/

Log:	Don't record/replay the reads from static, immutable structures (the
	RPython classes or the PBCs)

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -172,7 +172,8 @@
                             ('name', Ptr(rstr.STR)),
                             ('hash', Signed),
                             ('instantiate', Ptr(FuncType([], OBJECTPTR))),
-                            hints={'immutable': True}))
+                            hints={'immutable': True,
+                                   'static_immutable': True}))
 # non-gc case
 NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE))
 NONGCOBJECTPTR = Ptr(NONGCOBJECT)
@@ -273,7 +274,7 @@
         #
         self.rbase = getclassrepr(self.rtyper, self.classdef.basedef)
         self.rbase.setup()
-        kwds = {'hints': {'immutable': True}}
+        kwds = {'hints': {'immutable': True, 'static_immutable': True}}
         vtable_type = Struct('%s_vtable' % self.classdef.name,
                                 ('super', self.rbase.vtable_type),
                                 *llfields, **kwds)
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -243,7 +243,7 @@
         fields = []
         for row in self.uniquerows:
             fields.append((row.attrname, row.fntype))
-        kwds = {'hints': {'immutable': True}}
+        kwds = {'hints': {'immutable': True, 'static_immutable': True}}
         return Ptr(Struct('specfunc', *fields, **kwds))
 
     def create_specfunc(self):
@@ -658,7 +658,7 @@
     """For a SomePBC of frozen PBCs that have no common access set.
     The only possible operation on such a thing is comparison with 'is'."""
     lowleveltype = llmemory.Address
-    EMPTY = Struct('pbc', hints={'immutable': True})
+    EMPTY = Struct('pbc', hints={'immutable': True, 'static_immutable': True})
 
     def __init__(self, rtyper):
         self.rtyper = rtyper
@@ -719,7 +719,7 @@
 
     def _setup_repr(self):
         llfields = self._setup_repr_fields()
-        kwds = {'hints': {'immutable': True}}
+        kwds = {'hints': {'immutable': True, 'static_immutable': True}}
         self.pbc_type.become(Struct('pbc', *llfields, **kwds))
 
     def create_instance(self):
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -438,7 +438,8 @@
             result = '/* %s */' % result
         if self.db.reverse_debugger:
             S = self.lltypemap(op.args[0]).TO
-            if S._gckind != 'gc' and not S._hints.get('is_excdata'):
+            if (S._gckind != 'gc' and not S._hints.get('is_excdata')
+                    and not S._hints.get('static_immutable')):
                 from rpython.translator.revdb import revdb_genc
                 result = revdb_genc.emit(result, self.lltypename(op.result),
                                          newvalue)
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -40,7 +40,7 @@
         normal_code                                                     \
         {                                                               \
             decl_e = variable;                                          \
-            _RPY_REVDB_PRINT((stderr, "%s:%d: write %*llx\n",           \
+            _RPY_REVDB_PRINT((stderr, "%s:%d: write %0*llx\n",          \
                               __FILE__, __LINE__,                       \
                               2 * sizeof(_e), (unsigned long long)_e)); \
             memcpy(rpy_revdb.buf_p, &_e, sizeof(_e));                   \
@@ -57,7 +57,7 @@
             }                                                           \
             rpy_revdb.buf_p = _end1;                                    \
             memcpy(&_e, _src, sizeof(_e));                              \
-            _RPY_REVDB_PRINT((stderr, "%s:%d: read %*llx\n",            \
+            _RPY_REVDB_PRINT((stderr, "%s:%d: read %0*llx\n",           \
                               __FILE__, __LINE__,                       \
                               2 * sizeof(_e), (unsigned long long)_e)); \
             variable = _e;                                              \
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -127,6 +127,51 @@
         x = rdb.next('q'); assert x == 0      # number of stop points
         assert rdb.done()
 
+    def test_dont_record_vtable_reads(self):
+        class A(object):
+            x = 42
+        class B(A):
+            x = 43
+        lst = [A(), B()]
+        def main(argv):
+            print lst[len(argv) & 1].x
+            return 9
+        self.compile(main, [], backendopt=False)
+        out = self.run('Xx')
+        assert out == '42\n'
+        rdb = self.fetch_rdb()
+        rdb.read_check_argv([self.exename, 'Xx'])
+        # write() call (it used to be the case that vtable reads where
+        # recorded too; the single byte fetched from the vtable from
+        # the '.x' in main() would appear here)
+        x = rdb.next(); assert x == len(out)
+        x = rdb.next('i'); assert x == 0      # errno
+        # done
+        x = rdb.next('q'); assert x == 0      # number of stop points
+        assert rdb.done()
+
+    def test_dont_record_pbc_reads(self):
+        class MyPBC:
+            def _freeze_(self):
+                return True
+        pbc1 = MyPBC(); pbc1.x = 41
+        pbc2 = MyPBC(); pbc2.x = 42
+        lst = [pbc1, pbc2]
+        def main(argv):
+            print lst[len(argv) & 1].x
+            return 9
+        self.compile(main, [], backendopt=False)
+        out = self.run('Xx')
+        assert out == '41\n'
+        rdb = self.fetch_rdb()
+        rdb.read_check_argv([self.exename, 'Xx'])
+        # write() call
+        x = rdb.next(); assert x == len(out)
+        x = rdb.next('i'); assert x == 0      # errno
+        # done
+        x = rdb.next('q'); assert x == 0      # number of stop points
+        assert rdb.done()
+
 
 class InteractiveTests(object):
     EOF = pexpect.EOF

From pypy.commits at gmail.com  Sat Jun 11 06:55:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 03:55:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix for large inputs
Message-ID: <575bee34.264bc20a.5f60f.ffffd2bf@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85098:92150c86f642
Date: 2016-06-11 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/92150c86f642/

Log:	Fix for large inputs

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -606,12 +606,10 @@
         /* in the main process: continue reloading the revdb log */
         uint64_t delta = total_stop_points - rpy_revdb.stop_point_break;
         delta = (uint64_t)(delta * (1 - GOLDEN_RATIO));
-        if (delta == 0)
+        if (delta == 0 || frozen_num_pipes == NUM_FROZEN_PROCESSES - 1)
             rpy_revdb.stop_point_break = total_stop_points;
         else
             rpy_revdb.stop_point_break += delta;
-        if (rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)
-            rpy_revdb.stop_point_break++;
         close(fds[RD_SIDE]);
         fds[RD_SIDE] = -1;
     }
@@ -691,6 +689,8 @@
 {
     if (process_kind == PK_MAIN_PROCESS) {
         make_new_frozen_process();
+        if (process_kind == PK_MAIN_PROCESS)
+            return;
         if (rpy_revdb.stop_point_seen != rpy_revdb.stop_point_break)
             return;
     }

From pypy.commits at gmail.com  Sat Jun 11 07:48:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 04:48:02 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Force some more options to
 known-working values
Message-ID: <575bfa72.c61f1c0a.27c1f.35a8@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85099:eae582a6f933
Date: 2016-06-11 13:48 +0200
http://bitbucket.org/pypy/pypy/changeset/eae582a6f933/

Log:	Force some more options to known-working values

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -283,7 +283,9 @@
                default=False, cmdline='--revdb',
                requires=[('translation.split_gc_address_space', True),
                          ('translation.jit', False),
-                         ('translation.gc', 'boehm')]),
+                         ('translation.gc', 'boehm'),
+                         ('translation.thread', False),
+                         ('translation.continuation', False)]),
 ])
 
 def get_combined_translation_config(other_optdescr=None,

From pypy.commits at gmail.com  Sat Jun 11 08:09:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 05:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fixes
Message-ID: <575bff8f.0654c20a.1573.ffffe8f4@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85100:d36453797864
Date: 2016-06-11 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/d36453797864/

Log:	Fixes

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -28,6 +28,7 @@
 rpy_revdb_t rpy_revdb;
 static char rpy_rev_buffer[16384];
 static int rpy_rev_fileno = -1;
+static unsigned char flag_io_disabled;
 
 
 static void setup_record_mode(int argc, char *argv[]);
@@ -147,7 +148,10 @@
            we read it from the record.  In both cases, we cache the
            hash in the object, so that we record/replay only once per
            object. */
-        RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h);
+        if (flag_io_disabled)
+            h = ~((Signed)obj);
+        else
+            RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h);
         assert(h != 0);
         obj->h_hash = h;
     }
@@ -208,7 +212,6 @@
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
 static unsigned char process_kind = PK_MAIN_PROCESS;
 static unsigned char flag_exit_run_debug_process;
-static unsigned char flag_io_disabled;
 static jmp_buf jmp_buf_cancel_execution;
 static uint64_t latest_fork;
 
@@ -303,7 +306,7 @@
 }
 
 RPY_EXTERN
-char *rpy_reverse_db_fetch(int expected_size)
+char *rpy_reverse_db_fetch(int expected_size, const char *file, int line)
 {
     if (!flag_io_disabled) {
         ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p;
@@ -321,7 +324,8 @@
            running some custom code now, and we can't just perform I/O
            or access raw memory---because there is no raw memory! 
         */
-        printf("Attempted to do I/O or access raw memory\n");
+        printf("%s:%d: Attempted to do I/O or access raw memory\n",
+               file, line);
         longjmp(jmp_buf_cancel_execution, 1);
     }
 }
@@ -474,8 +478,9 @@
     }
     if (stop_points != rpy_revdb.stop_point_seen) {
         fprintf(stderr, "Bad number of stop points "
-                "(seen %llu, recorded %llu)\n", rpy_revdb.stop_point_seen,
-                stop_points);
+                "(seen %llu, recorded %llu)\n",
+                (unsigned long long)rpy_revdb.stop_point_seen,
+                (unsigned long long)stop_points);
         exit(1);
     }
     if (rpy_revdb.buf_p != rpy_revdb.buf_limit ||
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -52,7 +52,8 @@
             char *_src = rpy_revdb.buf_p;                               \
             char *_end1 = _src + sizeof(_e);                            \
             if (_end1 > rpy_revdb.buf_limit) {                          \
-                _src = rpy_reverse_db_fetch(sizeof(_e));                \
+                _src = rpy_reverse_db_fetch(sizeof(_e),                 \
+                                            __FILE__, __LINE__);        \
                 _end1 = _src + sizeof(_e);                              \
             }                                                           \
             rpy_revdb.buf_p = _end1;                                    \
@@ -77,7 +78,8 @@
     r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
 
 RPY_EXTERN void rpy_reverse_db_flush(void);
-RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size);
+RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
+                                      const char *file, int line);
 RPY_EXTERN void rpy_reverse_db_break(long stop_point);
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);

From pypy.commits at gmail.com  Sat Jun 11 15:22:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 12:22:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Catch exceptions from RPython
 commands
Message-ID: <575c6501.426dc20a.8e816.7349@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85101:2e1c0f9240a3
Date: 2016-06-11 21:23 +0200
http://bitbucket.org/pypy/pypy/changeset/2e1c0f9240a3/

Log:	Catch exceptions from RPython commands

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -284,6 +284,7 @@
                requires=[('translation.split_gc_address_space', True),
                          ('translation.jit', False),
                          ('translation.gc', 'boehm'),
+                         ("translation.rweakref", False),
                          ('translation.thread', False),
                          ('translation.continuation', False)]),
 ])
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -8,8 +8,9 @@
 #include 
 #include 
 
+#include "structdef.h"
+#include "forwarddecl.h"
 #include "preimpl.h"
-#include "structdef.h"
 #include "src/rtyper.h"
 #include "rdb-src/revdb_include.h"
 
@@ -367,8 +368,22 @@
     memcpy(_RPyString_AsString(s), arguments, length);
 
     disable_io(&dinfo);
-    if (setjmp(jmp_buf_cancel_execution) == 0)
+    if (setjmp(jmp_buf_cancel_execution) == 0) {
+        void *saved_t = pypy_g_ExcData.ed_exc_type;
+        void *saved_v = pypy_g_ExcData.ed_exc_value;
+        pypy_g_ExcData.ed_exc_type = NULL;
+        pypy_g_ExcData.ed_exc_value = NULL;
+
         rpy_revdb_command_funcs[index](s);
+
+        if (pypy_g_ExcData.ed_exc_type != NULL) {
+            printf("Command crashed with %.*s\n",
+                   (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
+                   pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
+        }
+        pypy_g_ExcData.ed_exc_type = saved_t;
+        pypy_g_ExcData.ed_exc_value = saved_v;
+    }
     enable_io(&dinfo);
 }
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -251,11 +251,23 @@
 
     def setup_class(cls):
         #
+        def g(cmdline):
+            if len(cmdline) > 5:
+                raise ValueError
+        g._dont_inline_ = True
+        #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
             if cmdline == 'oops':
                 for i in range(1000):
                     print 42     # I/O not permitted
+            if cmdline == 'raise-and-catch':
+                try:
+                    g(cmdline)
+                except ValueError:
+                    pass
+            if cmdline == 'crash':
+                raise ValueError
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -279,7 +291,8 @@
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r oops')
-        child.expectx('<<>>\r\nAttempted to do I/O or access raw memory')
+        child.expectx('<<>>\r\n')
+        child.expectx('Attempted to do I/O or access raw memory')
         child.expectx('(3)$ ')
 
     def test_interaction_with_forward(self):
@@ -288,8 +301,23 @@
         child.sendline('go 1')
         child.expectx('(1)$ ')
         child.sendline('r oops')
-        child.expectx('<<>>\r\nAttempted to do I/O or access raw memory')
+        child.expectx('<<>>\r\n')
+        child.expectx('Attempted to do I/O or access raw memory')
         child.expectx('(1)$ ')
         child.sendline('forward 50')
         child.expectx('At end.\r\n')
         child.expectx('(3)$ ')
+
+    def test_raise_and_catch(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r raise-and-catch')
+        child.expectx('<<>>\r\nblipped\r\n')
+        child.expectx('(3)$ ')
+
+    def test_crash(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r crash')
+        child.expectx('<<>>\r\nCommand crashed with ValueError')
+        child.expectx('(3)$ ')

From pypy.commits at gmail.com  Sat Jun 11 15:49:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 12:49:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Tweak the ratio
Message-ID: <575c6b55.c409c20a.ad0f2.6fe4@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85102:5a6c8b6f60a8
Date: 2016-06-11 21:50 +0200
http://bitbucket.org/pypy/pypy/changeset/5a6c8b6f60a8/

Log:	Tweak the ratio

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -200,7 +200,7 @@
 */
 
 #define NUM_FROZEN_PROCESSES   30
-#define GOLDEN_RATIO           0.618034
+#define STEP_RATIO             0.25
 
 #define RD_SIDE   0
 #define WR_SIDE   1
@@ -624,9 +624,14 @@
     }
     else {
         /* in the main process: continue reloading the revdb log */
-        uint64_t delta = total_stop_points - rpy_revdb.stop_point_break;
-        delta = (uint64_t)(delta * (1 - GOLDEN_RATIO));
-        if (delta == 0 || frozen_num_pipes == NUM_FROZEN_PROCESSES - 1)
+        uint64_t remaining = total_stop_points - rpy_revdb.stop_point_break;
+        uint64_t delta;
+        double step = STEP_RATIO;
+        int remaining_freezes = NUM_FROZEN_PROCESSES - frozen_num_pipes;
+        if (step * remaining_freezes < 1.0)
+            step = 1.0 / remaining_freezes;
+        delta = (uint64_t)(remaining * step);
+        if (delta == 0 || delta > remaining || remaining_freezes == 1)
             rpy_revdb.stop_point_break = total_stop_points;
         else
             rpy_revdb.stop_point_break += delta;

From pypy.commits at gmail.com  Sat Jun 11 18:04:18 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sat, 11 Jun 2016 15:04:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Handle call on iterable argument unpacking
Message-ID: <575c8ae2.45271c0a.cb2c7.fffff605@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85103:e67603023b8b
Date: 2016-06-12 00:02 +0200
http://bitbucket.org/pypy/pypy/changeset/e67603023b8b/

Log:	Handle call on iterable argument unpacking

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1055,14 +1055,21 @@
                             self.error("positional argument follows "
                                        "keyword argument unpacking",
                                        expr_node)
-                        else
+                        else:
                             self.error("positional argument follows "
                                        "keyword argument",
                                        expr_node)
                     args.append(self.handle_expr(expr_node))
-                elif expr_node.type == tokens.STAR
+                elif expr_node.type == tokens.STAR:
                     # an iterable argument unpacking
-                    # continue here
+                    if ndoublestars:
+                        self.error("iterable argument unpacking follows "
+                                   "keyword argument unpacking",
+                                   expr_node)
+                    expr = self.handle_expr(argument.get_child(1))
+                    args.append(ast.Starred(expr, ast.Load,
+                                            expr_node.get_lineno(),
+                                            expr_node.get_column()))
                 elif argument.get_child(1).type == syms.comp_for:
                     args.append(self.handle_genexp(argument))
                 else:

From pypy.commits at gmail.com  Sun Jun 12 02:46:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Jun 2016 23:46:27 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Write the argv in "clear text"
 inside the PYPYRDB log file, and
Message-ID: <575d0543.24f9c20a.1929.23f3@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85104:d1c588f9f886
Date: 2016-06-12 08:47 +0200
http://bitbucket.org/pypy/pypy/changeset/d1c588f9f886/

Log:	Write the argv in "clear text" inside the PYPYRDB log file, and
	display them again in --replay

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -14,12 +14,12 @@
 #include "src/rtyper.h"
 #include "rdb-src/revdb_include.h"
 
-#define RDB_SIGNATURE   0x0A424452    /* "RDB\n" */
+#define RDB_SIGNATURE   "RDB:"
 #define RDB_VERSION     0x00FF0001
 
 
 typedef struct {
-    Signed signature, version;
+    Signed version;
     Signed reserved1, reserved2;
     int argc;
     char **argv;
@@ -99,6 +99,7 @@
 {
     char *filename = getenv("PYPYRDB");
     rdb_header_t h;
+    int i;
 
     assert(RPY_RDB_REPLAY == 0);
     rpy_revdb.buf_p = rpy_rev_buffer;
@@ -115,8 +116,14 @@
         }
         atexit(rpy_reverse_db_flush);
 
+        write_all(RDB_SIGNATURE, strlen(RDB_SIGNATURE));
+        for (i = 0; i < argc; i++) {
+            write_all(" ", 1);
+            write_all(argv[i], strlen(argv[i]));
+        }
+        write_all("\n\0", 2);
+
         memset(&h, 0, sizeof(h));
-        h.signature = RDB_SIGNATURE;
         h.version = RDB_VERSION;
         h.argc = argc;
         h.argv = argv;
@@ -258,6 +265,8 @@
     char **argv = *argv_p;
     char *filename;
     rdb_header_t h;
+    char input[sizeof(rdb_header_t)];
+    ssize_t count;
 
     if (argc != 3) {
         fprintf(stderr, "syntax: %s --replay \n", argv[0]);
@@ -273,13 +282,23 @@
 
     assert(RPY_RDB_REPLAY == 1);
 
-    read_all(&h, sizeof(h));
-
-    if (h.signature != RDB_SIGNATURE) {
+    read_all(input, strlen(RDB_SIGNATURE));
+    if (strncmp(input, RDB_SIGNATURE, strlen(RDB_SIGNATURE)) != 0) {
         fprintf(stderr, "'%s' is not a RevDB file (or wrong platform)\n",
                 filename);
         exit(1);
     }
+    fprintf(stderr, "%s", RDB_SIGNATURE);
+    do {
+        count = read_at_least(input, 1, sizeof(input) - 1);
+        input[count] = 0;
+        fprintf(stderr, "%s", input);
+    } while (strlen(input) == count);
+
+    count -= (strlen(input) + 1);
+    memcpy(&h, input + strlen(input) + 1, count);
+
+    read_all(((char *)&h) + count, sizeof(h) - count);
     if (h.version != RDB_VERSION) {
         fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n",
                 (long)h.version, (long)RDB_VERSION);
@@ -288,10 +307,12 @@
     *argc_p = h.argc;
     *argv_p = h.argv;
 
-    if (lseek(rpy_rev_fileno, -sizeof(uint64_t), SEEK_END) < 0 ||
+    count = lseek(rpy_rev_fileno, 0, SEEK_CUR);
+    if (count < 0 ||
+            lseek(rpy_rev_fileno, -sizeof(uint64_t), SEEK_END) < 0 ||
             read(rpy_rev_fileno, &total_stop_points,
                  sizeof(uint64_t)) != sizeof(uint64_t) ||
-            lseek(rpy_rev_fileno, sizeof(h), SEEK_SET) != sizeof(h)) {
+            lseek(rpy_rev_fileno, count, SEEK_SET) != count) {
         fprintf(stderr, "%s: %m\n", filename);
         exit(1);
     }
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -14,17 +14,20 @@
 
 
 class RDB(object):
-    def __init__(self, filename):
+    def __init__(self, filename, expected_argv):
         with open(filename, 'rb') as f:
+            header = f.readline()
             self.buffer = f.read()
+        assert header == 'RDB: ' + ' '.join(expected_argv) + '\n'
         #
         self.cur = 0
-        x = self.next(); assert x == 0x0A424452
+        x = self.next('c'); assert x == '\x00'
         x = self.next(); assert x == 0x00FF0001
         x = self.next(); assert x == 0
         x = self.next(); assert x == 0
         self.argc = self.next()
         self.argv = self.next()
+        self.read_check_argv(expected_argv)
 
     def next(self, mode='P'):
         p = self.cur
@@ -81,8 +84,8 @@
     print >> sys.stderr, stderr
     return stdout
 
-def fetch_rdb(self):
-    return RDB(self.rdbname)
+def fetch_rdb(self, expected_argv):
+    return RDB(self.rdbname, map(str, expected_argv))
 
 
 class TestRecording(object):
@@ -96,8 +99,7 @@
             return 9
         self.compile(main, [], backendopt=False)
         assert self.run('abc d') == '[abc, d]\n'
-        rdb = self.fetch_rdb()
-        rdb.read_check_argv([self.exename, 'abc', 'd'])
+        rdb = self.fetch_rdb([self.exename, 'abc', 'd'])
         # write() call
         x = rdb.next(); assert x == len('[abc, d]\n')
         x = rdb.next('i'); assert x == 0      # errno
@@ -116,8 +118,7 @@
         match = re.match(r'\[(-?\d+), \1, \1]\n', out)
         assert match
         hash_value = int(match.group(1))
-        rdb = self.fetch_rdb()
-        rdb.read_check_argv([self.exename, 'Xx'])
+        rdb = self.fetch_rdb([self.exename, 'Xx'])
         # compute_identity_hash() call, but only the first one
         x = rdb.next(); assert intmask(x) == intmask(hash_value)
         # write() call
@@ -139,8 +140,7 @@
         self.compile(main, [], backendopt=False)
         out = self.run('Xx')
         assert out == '42\n'
-        rdb = self.fetch_rdb()
-        rdb.read_check_argv([self.exename, 'Xx'])
+        rdb = self.fetch_rdb([self.exename, 'Xx'])
         # write() call (it used to be the case that vtable reads where
         # recorded too; the single byte fetched from the vtable from
         # the '.x' in main() would appear here)
@@ -163,8 +163,7 @@
         self.compile(main, [], backendopt=False)
         out = self.run('Xx')
         assert out == '41\n'
-        rdb = self.fetch_rdb()
-        rdb.read_check_argv([self.exename, 'Xx'])
+        rdb = self.fetch_rdb([self.exename, 'Xx'])
         # write() call
         x = rdb.next(); assert x == len(out)
         x = rdb.next('i'); assert x == 0      # errno
@@ -198,7 +197,8 @@
             return 9
         compile(cls, main, [], backendopt=False)
         assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
-        assert fetch_rdb(cls).number_of_stop_points() == 3
+        rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef'])
+        assert rdb.number_of_stop_points() == 3
 
     def test_go(self):
         child = self.replay()

From pypy.commits at gmail.com  Sun Jun 12 10:46:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Jun 2016 07:46:59 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Ability to go forward
 programatically from RPython debug commands
Message-ID: <575d75e3.8f1d1c0a.7bc6a.ffffb53a@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85105:3d318752b5b2
Date: 2016-06-12 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/3d318752b5b2/

Log:	Ability to go forward programatically from RPython debug commands

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -567,6 +567,8 @@
 
     'revdb_stop_point':     LLOp(),
     'revdb_send_output':    LLOp(),
+    'revdb_go_forward':     LLOp(),
+    'revdb_get_value':      LLOp(sideeffects=False),
     'revdb_identityhash':   LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -14,7 +14,7 @@
 #include "src/rtyper.h"
 #include "rdb-src/revdb_include.h"
 
-#define RDB_SIGNATURE   "RDB:"
+#define RDB_SIGNATURE   "RevDB:"
 #define RDB_VERSION     0x00FF0001
 
 
@@ -41,7 +41,7 @@
 {
     /* init-time setup */
 
-    int replay_asked = (*argc_p >= 2 && !strcmp((*argv_p)[1], "--replay"));
+    int replay_asked = (*argc_p >= 2 && !strcmp((*argv_p)[1],"--revdb-replay"));
 
 #ifdef RPY_RDB_DYNAMIC_REPLAY
     RPY_RDB_REPLAY = replay_asked;
@@ -219,11 +219,11 @@
 
 enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
 static unsigned char process_kind = PK_MAIN_PROCESS;
-static unsigned char flag_exit_run_debug_process;
 static jmp_buf jmp_buf_cancel_execution;
-static uint64_t latest_fork;
+static uint64_t most_recent_fork;
+static uint64_t total_stop_points;
 
-static uint64_t total_stop_points;
+static void (*invoke_after_forward)(void);
 
 
 static void attach_gdb(void)
@@ -265,11 +265,11 @@
     char **argv = *argv_p;
     char *filename;
     rdb_header_t h;
-    char input[sizeof(rdb_header_t)];
+    char input[16];
     ssize_t count;
 
     if (argc != 3) {
-        fprintf(stderr, "syntax: %s --replay \n", argv[0]);
+        fprintf(stderr, "syntax: %s --revdb-replay \n", argv[0]);
         exit(2);
     }
     filename = argv[2];
@@ -289,16 +289,10 @@
         exit(1);
     }
     fprintf(stderr, "%s", RDB_SIGNATURE);
-    do {
-        count = read_at_least(input, 1, sizeof(input) - 1);
-        input[count] = 0;
-        fprintf(stderr, "%s", input);
-    } while (strlen(input) == count);
+    while ((read_all(input, 1), input[0] != 0))
+        fwrite(input, 1, 1, stderr);
 
-    count -= (strlen(input) + 1);
-    memcpy(&h, input + strlen(input) + 1, count);
-
-    read_all(((char *)&h) + count, sizeof(h) - count);
+    read_all(&h, sizeof(h));
     if (h.version != RDB_VERSION) {
         fprintf(stderr, "RevDB file version mismatch (got %lx, expected %lx)\n",
                 (long)h.version, (long)RDB_VERSION);
@@ -362,19 +356,25 @@
 
 static void enable_io(rpy_revdb_t *dinfo)
 {
+    uint64_t v;
     flag_io_disabled = 0;
+
+    /* restore the complete struct, with the exception of 'stop_point_break' */
+    v = rpy_revdb.stop_point_break;
     rpy_revdb = *dinfo;
+    rpy_revdb.stop_point_break = v;
 }
 
 /* generated by RPython */
 extern char *rpy_revdb_command_names[];
 extern void (*rpy_revdb_command_funcs[])(RPyString *);
 
+static void execute_rpy_function(void func(RPyString *), RPyString *arg);
+
 static void execute_rpy_command(long index, char *arguments)
 {
     size_t length = strlen(arguments);
     RPyString *s;
-    rpy_revdb_t dinfo;
 
     while (length > 0 && isspace(arguments[length - 1]))
         length--;
@@ -388,24 +388,32 @@
     RPyString_Size(s) = length;
     memcpy(_RPyString_AsString(s), arguments, length);
 
+    execute_rpy_function(rpy_revdb_command_funcs[index], s);
+}
+
+static void execute_rpy_function(void func(RPyString *), RPyString *arg)
+{
+    rpy_revdb_t dinfo;
+    void *saved_t = pypy_g_ExcData.ed_exc_type;
+    void *saved_v = pypy_g_ExcData.ed_exc_value;
+    pypy_g_ExcData.ed_exc_type = NULL;
+    pypy_g_ExcData.ed_exc_value = NULL;
     disable_io(&dinfo);
+    invoke_after_forward = NULL;
+
     if (setjmp(jmp_buf_cancel_execution) == 0) {
-        void *saved_t = pypy_g_ExcData.ed_exc_type;
-        void *saved_v = pypy_g_ExcData.ed_exc_value;
-        pypy_g_ExcData.ed_exc_type = NULL;
-        pypy_g_ExcData.ed_exc_value = NULL;
 
-        rpy_revdb_command_funcs[index](s);
+        func(arg);
 
         if (pypy_g_ExcData.ed_exc_type != NULL) {
             printf("Command crashed with %.*s\n",
                    (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
                    pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
         }
-        pypy_g_ExcData.ed_exc_type = saved_t;
-        pypy_g_ExcData.ed_exc_value = saved_v;
     }
     enable_io(&dinfo);
+    pypy_g_ExcData.ed_exc_type = saved_t;
+    pypy_g_ExcData.ed_exc_value = saved_v;
 }
 
 struct action_s {
@@ -533,6 +541,8 @@
         exit(1);
     }
 
+    fprintf(stderr, "\n");
+    fflush(stderr);
     printf("Replaying finished\n");
     printf("stop_points=%lld\n", (long long)stop_points);
 
@@ -580,7 +590,7 @@
             /* in the child: this is a debug process */
             process_kind = PK_DEBUG_PROCESS;
             assert(target_time >= rpy_revdb.stop_point_seen);
-            latest_fork = rpy_revdb.stop_point_seen;
+            most_recent_fork = rpy_revdb.stop_point_seen;
             rpy_revdb.stop_point_break = target_time;
             /* continue "running" the RPython program until we reach
                exactly the specified target_time */
@@ -616,7 +626,7 @@
         exit(1);
     }
 
-    fprintf(stderr, "forking at time %llu\n",
+    fprintf(stderr, "[%llu]",
             (unsigned long long)rpy_revdb.stop_point_seen);
 
     fds = frozen_pipes[frozen_num_pipes];
@@ -678,7 +688,7 @@
 
 static void act_info_fork(char *p)
 {
-    printf("latest_fork=%llu\n", (unsigned long long)latest_fork);
+    printf("most_recent_fork=%llu\n", (unsigned long long)most_recent_fork);
 }
 
 static void act_info(char *p)
@@ -703,7 +713,6 @@
         return;
     }
     rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + delta;
-    flag_exit_run_debug_process = 1;
 }
 
 static void run_debug_process(void)
@@ -716,17 +725,22 @@
         { "", act_nop },
         { NULL }
     };
-    flag_exit_run_debug_process = 0;
-    while (!flag_exit_run_debug_process) {
-        char input[256];
-        printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
-        fflush(stdout);
-        if (fgets(input, sizeof(input), stdin) != input) {
-            fprintf(stderr, "\n");
-            act_quit("");
-            abort();   /* unreachable */
+    while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
+        if (invoke_after_forward != NULL) {
+            execute_rpy_function((void(*)(RPyString *))invoke_after_forward,
+                                 NULL);
         }
-        process_input(input, "command", 1, actions_1);
+        else {
+            char input[256];
+            printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
+            fflush(stdout);
+            if (fgets(input, sizeof(input), stdin) != input) {
+                fprintf(stderr, "\n");
+                act_quit("");
+                abort();   /* unreachable */
+            }
+            process_input(input, "command", 1, actions_1);
+        }
     }
 }
 
@@ -737,8 +751,6 @@
         make_new_frozen_process();
         if (process_kind == PK_MAIN_PROCESS)
             return;
-        if (rpy_revdb.stop_point_seen != rpy_revdb.stop_point_break)
-            return;
     }
     assert(process_kind == PK_DEBUG_PROCESS);
     run_debug_process();
@@ -750,5 +762,31 @@
     fwrite(_RPyString_AsString(output), 1, RPyString_Size(output), stdout);
 }
 
+RPY_EXTERN
+void rpy_reverse_db_go_forward(Signed steps, void callback(void))
+{
+    if (steps < 0) {
+        fprintf(stderr, "revdb.go_forward(): negative amount of steps\n");
+        exit(1);
+    }
+    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + steps;
+    invoke_after_forward = callback;
+}
+
+RPY_EXTERN
+Signed rpy_reverse_db_get_value(char value_id)
+{
+    switch (value_id) {
+    case 'c':       /* current_time() */
+        return rpy_revdb.stop_point_seen;
+    case 'm':       /* most_recent_fork() */
+        return most_recent_fork;
+    case 't':       /* total_time() */
+        return total_stop_points;
+    default:
+        return -1;
+    }
+}
+
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -74,6 +74,12 @@
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
 
+#define OP_REVDB_GO_FORWARD(time_delta, callback, r)                    \
+    rpy_reverse_db_go_forward(time_delta, callback)
+
+#define OP_REVDB_GET_VALUE(value_id, r)                                 \
+    r = rpy_reverse_db_get_value(value_id)
+
 #define OP_REVDB_IDENTITYHASH(obj, r)                                   \
     r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
 
@@ -83,6 +89,8 @@
 RPY_EXTERN void rpy_reverse_db_break(long stop_point);
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
+RPY_EXTERN void rpy_reverse_db_go_forward(Signed steps, void callback(void));
+RPY_EXTERN Signed rpy_reverse_db_get_value(char value_id);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -18,7 +18,7 @@
         with open(filename, 'rb') as f:
             header = f.readline()
             self.buffer = f.read()
-        assert header == 'RDB: ' + ' '.join(expected_argv) + '\n'
+        assert header == 'RevDB: ' + ' '.join(expected_argv) + '\n'
         #
         self.cur = 0
         x = self.next('c'); assert x == '\x00'
@@ -178,7 +178,7 @@
     def replay(self, **kwds):
         kwds.setdefault('timeout', 10)
         child = pexpect.spawn(str(self.exename),
-                              ['--replay', str(self.rdbname)], **kwds)
+                              ['--revdb-replay', str(self.rdbname)], **kwds)
         child.logfile = sys.stdout
         def expectx(s):
             child.expect(re.escape(s))
@@ -202,8 +202,8 @@
 
     def test_go(self):
         child = self.replay()
-        child.expectx('stop_points=3\r\n')
-        child.expectx('(3)$ ')
+        child.expectx('stop_points=3\r\n'
+                      '(3)$ ')
         child.sendline('go 1')
         child.expectx('(1)$ ')
         child.sendline('')
@@ -223,7 +223,7 @@
     def test_info_fork(self):
         child = self.replay()
         child.sendline('info fork')
-        child.expectx('latest_fork=3\r\n')
+        child.expectx('most_recent_fork=3\r\n')
 
     def test_quit(self):
         child = self.replay()
@@ -239,12 +239,12 @@
         child.sendline('forward 1')
         child.expectx('(3)$ ')
         child.sendline('info fork')
-        child.expectx('latest_fork=1\r\n')
+        child.expectx('most_recent_fork=1\r\n')
         child.sendline('forward 1')
-        child.expectx('At end.\r\n')
-        child.expectx('(3)$ ')
+        child.expectx('At end.\r\n'
+                      '(3)$ ')
         child.sendline('info fork')
-        child.expectx('latest_fork=3\r\n')
+        child.expectx('most_recent_fork=3\r\n')
 
 
 class TestDebugCommands(InteractiveTests):
@@ -256,6 +256,11 @@
                 raise ValueError
         g._dont_inline_ = True
         #
+        def went_fw():
+            revdb.send_output('went-fw -> %d\n' % revdb.current_time())
+            if revdb.current_time() != revdb.total_time():
+                revdb.go_forward(1, went_fw)
+        #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
             if cmdline == 'oops':
@@ -268,6 +273,12 @@
                     pass
             if cmdline == 'crash':
                 raise ValueError
+            if cmdline == 'get-value':
+                revdb.send_output('%d,%d,%d\n' % (revdb.current_time(),
+                                                  revdb.most_recent_fork(),
+                                                  revdb.total_time()))
+            if cmdline == 'go-fw':
+                revdb.go_forward(1, went_fw)
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -284,16 +295,17 @@
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r  foo  bar  baz  ')
-        child.expectx('<<>>\r\nblipped\r\n')
-        child.expectx('(3)$ ')
+        child.expectx('<<>>\r\n'
+                      'blipped\r\n'
+                      '(3)$ ')
 
     def test_io_not_permitted(self):
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r oops')
         child.expectx('<<>>\r\n')
-        child.expectx('Attempted to do I/O or access raw memory')
-        child.expectx('(3)$ ')
+        child.expectx(': Attempted to do I/O or access raw memory\r\n'
+                      '(3)$ ')
 
     def test_interaction_with_forward(self):
         child = self.replay()
@@ -302,22 +314,47 @@
         child.expectx('(1)$ ')
         child.sendline('r oops')
         child.expectx('<<>>\r\n')
-        child.expectx('Attempted to do I/O or access raw memory')
-        child.expectx('(1)$ ')
+        child.expectx('Attempted to do I/O or access raw memory\r\n'
+                      '(1)$ ')
         child.sendline('forward 50')
-        child.expectx('At end.\r\n')
-        child.expectx('(3)$ ')
+        child.expectx('At end.\r\n'
+                      '(3)$ ')
 
     def test_raise_and_catch(self):
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r raise-and-catch')
-        child.expectx('<<>>\r\nblipped\r\n')
-        child.expectx('(3)$ ')
+        child.expectx('<<>>\r\n'
+                      'blipped\r\n'
+                      '(3)$ ')
 
     def test_crash(self):
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r crash')
-        child.expectx('<<>>\r\nCommand crashed with ValueError')
+        child.expectx('<<>>\r\n'
+                      'Command crashed with ValueError\r\n'
+                      '(3)$ ')
+
+    def test_get_value(self):
+        child = self.replay()
         child.expectx('(3)$ ')
+        child.sendline('go 2')
+        child.expectx('(2)$ ')
+        child.sendline('r get-value')
+        child.expectx('<<>>\r\n'
+                      '2,1,3\r\n'
+                      'blipped\r\n'
+                      '(2)$ ')
+
+    def test_go_fw(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r go-fw')
+        child.expectx('<<>>\r\n'
+                      'blipped\r\n'
+                      'went-fw -> 2\r\n'
+                      'went-fw -> 3\r\n'
+                      '(3)$ ')

From pypy.commits at gmail.com  Sun Jun 12 10:59:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Jun 2016 07:59:54 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add a forgotten file
Message-ID: <575d78ea.4e4ec20a.a91f1.24d9@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85106:c7f7ae742625
Date: 2016-06-12 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c7f7ae742625/

Log:	Add a forgotten file

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/revdb.py
@@ -0,0 +1,58 @@
+import sys
+from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import lltype, rstr
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rtyper.annlowlevel import llhelper
+
+
+def stop_point(n):
+    if we_are_translated():
+        if fetch_translated_config().translation.reverse_debugger:
+            llop.revdb_stop_point(lltype.Void, n)
+
+def register_debug_command(command, lambda_func):
+    pass
+
+def send_output(string):      # monkey-patch this for untranslated tests
+    llop.revdb_send_output(lltype.Void, string)
+
+def current_time():
+    return llop.revdb_get_value(lltype.Signed, 'c')
+
+def most_recent_fork():
+    return llop.revdb_get_value(lltype.Signed, 'm')
+
+def total_time():
+    return llop.revdb_get_value(lltype.Signed, 't')
+
+ at specialize.arg(1)
+def go_forward(time_delta, callback):
+    ll_callback = llhelper(_CALLBACK_FNPTR, callback)
+    llop.revdb_go_forward(lltype.Void, time_delta, ll_callback)
+_CALLBACK_FNPTR = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+
+class RegisterDebugCommand(ExtRegistryEntry):
+    _about_ = register_debug_command
+
+    def compute_result_annotation(self, s_command, s_lambda_func):
+        from rpython.annotator import model as annmodel
+        command = s_command.const
+        lambda_func = s_lambda_func.const
+        assert isinstance(command, str)
+        t = self.bookkeeper.annotator.translator
+        if t.config.translation.reverse_debugger:
+            func = lambda_func()
+            try:
+                cmds = t.revdb_commands
+            except AttributeError:
+                cmds = t.revdb_commands = {}
+            cmds[command] = func
+            s_func = self.bookkeeper.immutablevalue(func)
+            self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
+                                             s_func, [annmodel.s_Str0])
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()

From pypy.commits at gmail.com  Sun Jun 12 12:15:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Jun 2016 09:15:20 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add the jump_in_time() function
Message-ID: <575d8a98.4d9a1c0a.b0a91.ffff8609@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85107:535af90a99e0
Date: 2016-06-12 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/535af90a99e0/

Log:	Add the jump_in_time() function

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -4,34 +4,75 @@
 from rpython.rtyper.lltypesystem import lltype, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.annlowlevel import llhelper
+from rpython.rtyper.annlowlevel import llhelper, hlstr
 
 
 def stop_point(n):
+    """Indicates a point in the execution of the RPython program where
+    the reverse-debugger can stop.  When reverse-debugging, we see
+    the "time" as the index of the stop-point that happened.
+    """
     if we_are_translated():
         if fetch_translated_config().translation.reverse_debugger:
             llop.revdb_stop_point(lltype.Void, n)
 
 def register_debug_command(command, lambda_func):
-    pass
+    """Register the extra RPython-implemented debug command."""
 
-def send_output(string):      # monkey-patch this for untranslated tests
+def send_output(string):
+    """For RPython debug commands: writes the string to stdout."""
     llop.revdb_send_output(lltype.Void, string)
 
 def current_time():
+    """For RPython debug commands: returns the current time."""
     return llop.revdb_get_value(lltype.Signed, 'c')
 
 def most_recent_fork():
+    """For RPython debug commands: returns the time of the most
+    recent fork.  Going back to that time is fast; going back to a time
+    just before is slow."""
     return llop.revdb_get_value(lltype.Signed, 'm')
 
 def total_time():
+    """For RPython debug commands: returns the total time (measured
+    as the total number of stop-points)."""
     return llop.revdb_get_value(lltype.Signed, 't')
 
 @specialize.arg(1)
-def go_forward(time_delta, callback):
-    ll_callback = llhelper(_CALLBACK_FNPTR, callback)
-    llop.revdb_go_forward(lltype.Void, time_delta, ll_callback)
-_CALLBACK_FNPTR = lltype.Ptr(lltype.FuncType([], lltype.Void))
+def go_forward(time_delta, callback, arg_string):
+    """For RPython debug commands: tells that after this function finishes,
+    the debugger should run the 'forward ' command and then
+    invoke the 'callback' with no argument.
+    """
+    _change_time('f', time_delta, callback, arg_string)
+
+ at specialize.arg(1)
+def jump_in_time(target_time, callback, arg_string):
+    """For RPython debug commands: the debugger should run the
+    'go ' command.  This will reset the memory and fork again,
+    so you can't save any RPython state and read it back.  You can only
+    encode the state you want to save into a string.  In the reloaded
+    process, 'callback(arg_string)' is called.
+    """
+    _change_time('g', target_time, callback, arg_string)
+
+
+# ____________________________________________________________
+
+
+ at specialize.arg(1)
+def _change_time(mode, time, callback, arg_string):
+    callback_wrapper = _make_callback(callback)
+    ll_callback = llhelper(_CALLBACK_ARG_FNPTR, callback_wrapper)
+    llop.revdb_change_time(lltype.Void, mode, time, ll_callback, arg_string)
+
+ at specialize.memo()
+def _make_callback(callback):
+    def callback_wrapper(ll_string):
+        callback(hlstr(ll_string))
+    return callback_wrapper
+_CALLBACK_ARG_FNPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)],
+                                                 lltype.Void))
 
 
 class RegisterDebugCommand(ExtRegistryEntry):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -567,7 +567,7 @@
 
     'revdb_stop_point':     LLOp(),
     'revdb_send_output':    LLOp(),
-    'revdb_go_forward':     LLOp(),
+    'revdb_change_time':    LLOp(),
     'revdb_get_value':      LLOp(sideeffects=False),
     'revdb_identityhash':   LLOp(),
 }
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -223,7 +223,14 @@
 static uint64_t most_recent_fork;
 static uint64_t total_stop_points;
 
-static void (*invoke_after_forward)(void);
+static void (*invoke_after_forward)(RPyString *);
+static RPyString *invoke_argument;
+
+struct jump_in_time_s {
+    uint64_t target_time;
+    void *callback;
+    size_t arg_length;
+};
 
 
 static void attach_gdb(void)
@@ -369,6 +376,20 @@
 extern char *rpy_revdb_command_names[];
 extern void (*rpy_revdb_command_funcs[])(RPyString *);
 
+static RPyString *make_rpy_string(size_t length)
+{
+    RPyString *s = malloc(sizeof(RPyString) + length);
+    if (s == NULL) {
+        fprintf(stderr, "out of memory for a string of %llu chars\n",
+                (unsigned long long)length);
+        exit(1);
+    }
+    /* xxx assumes Boehm here for now */
+    memset(s, 0, sizeof(RPyString));
+    RPyString_Size(s) = length;
+    return s;
+}
+
 static void execute_rpy_function(void func(RPyString *), RPyString *arg);
 
 static void execute_rpy_command(long index, char *arguments)
@@ -378,14 +399,7 @@
 
     while (length > 0 && isspace(arguments[length - 1]))
         length--;
-    s = malloc(sizeof(RPyString) + length);
-    if (s == NULL) {
-        fprintf(stderr, "out of memory\n");
-        exit(1);
-    }
-    /* xxx assumes Boehm here for now */
-    memset(s, 0, sizeof(RPyString));
-    RPyString_Size(s) = length;
+    s = make_rpy_string(length);
     memcpy(_RPyString_AsString(s), arguments, length);
 
     execute_rpy_function(rpy_revdb_command_funcs[index], s);
@@ -400,6 +414,7 @@
     pypy_g_ExcData.ed_exc_value = NULL;
     disable_io(&dinfo);
     invoke_after_forward = NULL;
+    invoke_argument = NULL;
 
     if (setjmp(jmp_buf_cancel_execution) == 0) {
 
@@ -496,22 +511,46 @@
     return 0;
 }
 
-static void cmd_go(uint64_t target_time)
+static int copy_pipe(int dst_fd, int src_fd, ssize_t count)
 {
+    char buffer[16384];
+    while (count > 0) {
+        ssize_t count1 = count > sizeof(buffer) ? sizeof(buffer) : count;
+        if (read_pipe(src_fd, buffer, count1) < 0 ||
+            write_pipe(dst_fd, buffer, count1) < 0)
+            return -1;
+        count -= count1;
+    }
+    return 0;
+}
+
+static void cmd_go(uint64_t target_time, void callback(RPyString *),
+                   RPyString *arg)
+{
+    struct jump_in_time_s header;
+
+    header.target_time = target_time;
+    header.callback = callback;    /* may be NULL */
+    /* ^^^ assumes the fn address is the same in the various forks */
+    header.arg_length = arg == NULL ? 0 : RPyString_Size(arg);
+
     assert(process_kind == PK_DEBUG_PROCESS);
-    write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
-               sizeof(target_time));
+    write_pipe(frozen_pipe_signal[WR_SIDE], &header, sizeof(header));
+    if (header.arg_length > 0) {
+        write_pipe(frozen_pipe_signal[WR_SIDE], _RPyString_AsString(arg),
+                   header.arg_length);
+    }
     exit(0);
 }
 
 static void check_at_end(uint64_t stop_points)
 {
     char dummy[1];
-    uint64_t target_time;
+    struct jump_in_time_s jump_in_time;
 
     if (process_kind == PK_DEBUG_PROCESS) {
         printf("At end.\n");
-        cmd_go(rpy_revdb.stop_point_seen);
+        cmd_go(rpy_revdb.stop_point_seen, NULL, NULL);
         abort();   /* unreachable */
     }
 
@@ -549,22 +588,27 @@
     close(frozen_pipe_signal[WR_SIDE]);
     frozen_pipe_signal[WR_SIDE] = -1;
 
-    target_time = frozen_time[frozen_num_pipes-1];
-    while (target_time != (uint64_t)-1) {
+    memset(&jump_in_time, 0, sizeof(jump_in_time));
+    jump_in_time.target_time = frozen_time[frozen_num_pipes-1];
+
+    while (jump_in_time.target_time != (uint64_t)-1) {
         int p = frozen_num_pipes - 1;
-        if (target_time > frozen_time[p])
-            target_time = frozen_time[p];
-        while (frozen_time[p] > target_time)
+        if (jump_in_time.target_time > frozen_time[p])
+            jump_in_time.target_time = frozen_time[p];
+        while (frozen_time[p] > jump_in_time.target_time)
             p--;
         if (write_pipe(frozen_pipes[p][WR_SIDE],
-                       &target_time, sizeof(target_time)) < 0) {
+                       &jump_in_time, sizeof(jump_in_time)) < 0 ||
+            copy_pipe(frozen_pipes[p][WR_SIDE],
+                      frozen_pipe_signal[RD_SIDE],
+                      jump_in_time.arg_length) < 0) {
             fprintf(stderr, "broken pipe to frozen subprocess\n");
             exit(1);
         }
         /* blocking here while the p'th frozen process spawns a debug process
            and the user interacts with it; then: */
-        if (read_pipe(frozen_pipe_signal[RD_SIDE], &target_time,
-                      sizeof(target_time)) < 0) {
+        if (read_pipe(frozen_pipe_signal[RD_SIDE], &jump_in_time,
+                      sizeof(jump_in_time)) < 0) {
             fprintf(stderr, "broken signal pipe\n");
             exit(1);
         }
@@ -574,11 +618,11 @@
 
 static void run_frozen_process(int frozen_pipe_fd)
 {
-    uint64_t target_time;
+    struct jump_in_time_s jump_in_time;
     pid_t child_pid;
 
     while (1) {
-        if (read_pipe(frozen_pipe_fd, &target_time, sizeof(target_time)) < 0)
+        if (read_pipe(frozen_pipe_fd, &jump_in_time, sizeof(jump_in_time)) < 0)
             exit(1);
 
         child_pid = fork();
@@ -589,9 +633,24 @@
         if (child_pid == 0) {
             /* in the child: this is a debug process */
             process_kind = PK_DEBUG_PROCESS;
-            assert(target_time >= rpy_revdb.stop_point_seen);
+            assert(jump_in_time.target_time >= rpy_revdb.stop_point_seen);
             most_recent_fork = rpy_revdb.stop_point_seen;
-            rpy_revdb.stop_point_break = target_time;
+            rpy_revdb.stop_point_break = jump_in_time.target_time;
+
+            if (jump_in_time.callback == NULL) {
+                assert(jump_in_time.arg_length == 0);
+                assert(invoke_after_forward == NULL);
+            }
+            else {
+                RPyString *s = make_rpy_string(jump_in_time.arg_length);
+                if (read_pipe(frozen_pipe_fd, _RPyString_AsString(s),
+                              jump_in_time.arg_length) < 0) {
+                    fprintf(stderr, "broken pipe to debug subprocess\n");
+                    exit(1);
+                }
+                invoke_after_forward = jump_in_time.callback;
+                invoke_argument = s;
+            }
             /* continue "running" the RPython program until we reach
                exactly the specified target_time */
             break;
@@ -605,11 +664,9 @@
             if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
                 ;     /* normal exit */
             else {
-                target_time = (uint64_t)-1;
                 fprintf(stderr, "debugging subprocess died\n");
-                write_pipe(frozen_pipe_signal[WR_SIDE], &target_time,
-                           sizeof(target_time));
-                exit(1);    /* error */
+                cmd_go((uint64_t)-1, NULL, NULL);
+                abort();    /* unreachable */
             }
         }
     }
@@ -673,7 +730,7 @@
 
 static void act_quit(char *p)
 {
-    cmd_go((uint64_t)-1);
+    cmd_go((uint64_t)-1, NULL, NULL);
 }
 
 static void act_go(char *p)
@@ -683,7 +740,7 @@
         printf("usage: go \n");
         return;
     }
-    cmd_go(target_time);
+    cmd_go(target_time, NULL, NULL);
 }
 
 static void act_info_fork(char *p)
@@ -727,8 +784,7 @@
     };
     while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
         if (invoke_after_forward != NULL) {
-            execute_rpy_function((void(*)(RPyString *))invoke_after_forward,
-                                 NULL);
+            execute_rpy_function(invoke_after_forward, invoke_argument);
         }
         else {
             char input[256];
@@ -763,14 +819,28 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_go_forward(Signed steps, void callback(void))
+void rpy_reverse_db_change_time(char mode, Signed time,
+                                void callback(RPyString *), RPyString *arg)
 {
-    if (steps < 0) {
-        fprintf(stderr, "revdb.go_forward(): negative amount of steps\n");
-        exit(1);
+    switch (mode) {
+
+    case 'f': {      /* forward */
+        if (time < 0) {
+            fprintf(stderr, "revdb.go_forward(): negative amount of steps\n");
+            exit(1);
+        }
+        rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + time;
+        invoke_after_forward = callback;
+        invoke_argument = arg;
+        break;
     }
-    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + steps;
-    invoke_after_forward = callback;
+    case 'g': {      /* go */
+        cmd_go(time >= 1 ? time : 1, callback, arg);
+        abort();    /* unreachable */
+    }
+    default:
+        abort();    /* unreachable */
+    }
 }
 
 RPY_EXTERN
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/rdb-src/revdb_include.h
--- a/rpython/translator/revdb/rdb-src/revdb_include.h
+++ b/rpython/translator/revdb/rdb-src/revdb_include.h
@@ -74,8 +74,8 @@
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
 
-#define OP_REVDB_GO_FORWARD(time_delta, callback, r)                    \
-    rpy_reverse_db_go_forward(time_delta, callback)
+#define OP_REVDB_CHANGE_TIME(mode, time, callback, ll_string, r)   \
+    rpy_reverse_db_change_time(mode, time, callback, ll_string)
 
 #define OP_REVDB_GET_VALUE(value_id, r)                                 \
     r = rpy_reverse_db_get_value(value_id)
@@ -89,7 +89,9 @@
 RPY_EXTERN void rpy_reverse_db_break(long stop_point);
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
-RPY_EXTERN void rpy_reverse_db_go_forward(Signed steps, void callback(void));
+RPY_EXTERN void rpy_reverse_db_change_time(char mode, Signed time,
+                                           void callback(RPyString *),
+                                           RPyString *arg);
 RPY_EXTERN Signed rpy_reverse_db_get_value(char value_id);
 
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -256,10 +256,16 @@
                 raise ValueError
         g._dont_inline_ = True
         #
-        def went_fw():
-            revdb.send_output('went-fw -> %d\n' % revdb.current_time())
+        def went_fw(arg):
+            revdb.send_output('went-fw %s -> %d\n' % (arg,
+                                                      revdb.current_time()))
             if revdb.current_time() != revdb.total_time():
-                revdb.go_forward(1, went_fw)
+                revdb.go_forward(1, went_fw, "yy")
+        def changed_time(arg):
+            revdb.send_output('changed-time %s -> %d\n' % (arg,
+                                                      revdb.current_time()))
+            if revdb.current_time() != revdb.total_time():
+                revdb.go_forward(1, went_fw, "zz")
         #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
@@ -278,7 +284,9 @@
                                                   revdb.most_recent_fork(),
                                                   revdb.total_time()))
             if cmdline == 'go-fw':
-                revdb.go_forward(1, went_fw)
+                revdb.go_forward(1, went_fw, "xx")
+            if cmdline == 'change-time':
+                revdb.jump_in_time(2, changed_time, "xyzzy")
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -355,6 +363,15 @@
         child.sendline('r go-fw')
         child.expectx('<<>>\r\n'
                       'blipped\r\n'
-                      'went-fw -> 2\r\n'
-                      'went-fw -> 3\r\n'
+                      'went-fw xx -> 2\r\n'
+                      'went-fw yy -> 3\r\n'
                       '(3)$ ')
+
+    def test_change_time(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r change-time')
+        child.expectx('<<>>\r\n'
+                      'changed-time xyzzy -> 2\r\n'
+                      'went-fw zz -> 3\r\n'
+                      '(3)$ ')

From pypy.commits at gmail.com  Sun Jun 12 12:23:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Jun 2016 09:23:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix
Message-ID: <575d8c94.81261c0a.31e22.29da@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85108:7981f3915c95
Date: 2016-06-12 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/7981f3915c95/

Log:	fix

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -60,7 +60,7 @@
 # ____________________________________________________________
 
 
- at specialize.arg(1)
+ at specialize.arg(2)
 def _change_time(mode, time, callback, arg_string):
     callback_wrapper = _make_callback(callback)
     ll_callback = llhelper(_CALLBACK_ARG_FNPTR, callback_wrapper)

From pypy.commits at gmail.com  Sun Jun 12 13:11:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Jun 2016 10:11:37 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: 'forward' is potentially
 dangerous if used after RPython commands mucked
Message-ID: <575d97c9.aa29c20a.20e25.6b4e@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85109:9524af3f0d05
Date: 2016-06-12 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/9524af3f0d05/

Log:	'forward' is potentially dangerous if used after RPython commands
	mucked with objects. rename '__forward'

diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/rdb-src/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/rdb-src/revdb.c
@@ -776,9 +776,9 @@
 {
     static struct action_s actions_1[] = {
         { "go", act_go },
-        { "forward", act_forward },
         { "info", act_info },
         { "quit", act_quit },
+        { "__forward", act_forward },
         { "", act_nop },
         { NULL }
     };
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -234,13 +234,13 @@
         child = self.replay()
         child.sendline('go 1')
         child.expectx('(1)$ ')
-        child.sendline('forward 1')
+        child.sendline('__forward 1')
         child.expectx('(2)$ ')
-        child.sendline('forward 1')
+        child.sendline('__forward 1')
         child.expectx('(3)$ ')
         child.sendline('info fork')
         child.expectx('most_recent_fork=1\r\n')
-        child.sendline('forward 1')
+        child.sendline('__forward 1')
         child.expectx('At end.\r\n'
                       '(3)$ ')
         child.sendline('info fork')
@@ -324,7 +324,7 @@
         child.expectx('<<>>\r\n')
         child.expectx('Attempted to do I/O or access raw memory\r\n'
                       '(1)$ ')
-        child.sendline('forward 50')
+        child.sendline('__forward 50')
         child.expectx('At end.\r\n'
                       '(3)$ ')
 

From pypy.commits at gmail.com  Sun Jun 12 13:34:40 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 12 Jun 2016 10:34:40 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: factor out some code
Message-ID: <575d9d30.0148c20a.49abc.ffff9836@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85110:f912e70460d8
Date: 2016-06-11 17:34 +0100
http://bitbucket.org/pypy/pypy/changeset/f912e70460d8/

Log:	factor out some code

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -31,6 +31,16 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
+def create_so(modname, **kwds):
+    eci = ExternalCompilationInfo(**kwds)
+    eci = eci.convert_sources_to_files()
+    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+    soname = platform.platform.compile(
+        [], eci,
+        outputfilename=str(dirname/modname),
+        standalone=False)
+    return soname
+
 def compile_extension_module(space, modname, include_dirs=[], **kwds):
     """
     Build an extension module and return the filename of the resulting native
@@ -57,20 +67,12 @@
     else:
         kwds["link_files"] = [str(api_library + '.so')]
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
-            kwds["link_extra"]=["-g"]
+            kwds["compile_extra"] = ["-Werror", "-g", "-O0"]
+            kwds["link_extra"] = ["-g"]
+    kwds['include_dirs'] = api.include_dirs + include_dirs
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs=api.include_dirs + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname, **kwds)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
@@ -94,18 +96,10 @@
         pass
     elif sys.platform.startswith('linux'):
             kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"]
+    kwds['include_dirs'] = [space.include_dir] + include_dirs
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs = [space.include_dir] + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname, **kwds)
     return str(soname)
 
 def freeze_refcnts(self):

From pypy.commits at gmail.com  Sun Jun 12 13:34:42 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 12 Jun 2016 10:34:42 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Don't paste in RPython defs
 when compiling Python extensions in cpyext tests
Message-ID: <575d9d32.03a81c0a.4267e.2151@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85111:ae7a98d56f75
Date: 2016-06-12 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/ae7a98d56f75/

Log:	Don't paste in RPython defs when compiling Python extensions in
	cpyext tests

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -31,10 +31,28 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
-def create_so(modname, **kwds):
-    eci = ExternalCompilationInfo(**kwds)
-    eci = eci.convert_sources_to_files()
+def convert_sources_to_files(sources, dirname):
+    files = []
+    for i, source in enumerate(sources):
+        filename = dirname / ('source_%d.c' % i)
+        with filename.open('w') as f:
+            f.write(str(source))
+        files.append(filename)
+    return files
+
+def create_so(modname, include_dirs,
+        separate_module_sources=None,
+        separate_module_files=None,
+        **kwds):
     dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+    if separate_module_sources:
+        assert not separate_module_files
+        files = convert_sources_to_files(separate_module_sources, dirname)
+        separate_module_files = files
+    eci = ExternalCompilationInfo(
+        include_dirs=include_dirs,
+        separate_module_files=separate_module_files,
+        **kwds)
     soname = platform.platform.compile(
         [], eci,
         outputfilename=str(dirname/modname),
@@ -69,10 +87,11 @@
         if sys.platform.startswith('linux'):
             kwds["compile_extra"] = ["-Werror", "-g", "-O0"]
             kwds["link_extra"] = ["-g"]
-    kwds['include_dirs'] = api.include_dirs + include_dirs
 
     modname = modname.split('.')[-1]
-    soname = create_so(modname, **kwds)
+    soname = create_so(modname,
+            include_dirs=api.include_dirs + include_dirs,
+            **kwds)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
@@ -95,11 +114,12 @@
     elif sys.platform == 'darwin':
         pass
     elif sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"]
-    kwds['include_dirs'] = [space.include_dir] + include_dirs
+        kwds["compile_extra"] = ["-O0", "-g","-Werror=implicit-function-declaration"]
 
     modname = modname.split('.')[-1]
-    soname = create_so(modname, **kwds)
+    soname = create_so(modname,
+            include_dirs=[space.include_dir] + include_dirs,
+            **kwds)
     return str(soname)
 
 def freeze_refcnts(self):
@@ -294,6 +314,11 @@
                 /* fix for cpython 2.7 Python.h if running tests with -A
                    since pypy compiles with -fvisibility-hidden */
                 #undef PyMODINIT_FUNC
+                #ifdef __GNUC__
+                #  define RPY_EXPORTED extern __attribute__((visibility("default")))
+                #else
+                #  define RPY_EXPORTED extern __declspec(dllexport)
+                #endif
                 #define PyMODINIT_FUNC RPY_EXPORTED void
 
                 %(body)s

From pypy.commits at gmail.com  Sun Jun 12 14:21:25 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sun, 12 Jun 2016 11:21:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Handle keyword unpacking in call
Message-ID: <575da825.4275c20a.63b1e.ffffdb6e@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85112:3dc8310f4934
Date: 2016-06-12 20:20 +0200
http://bitbucket.org/pypy/pypy/changeset/3dc8310f4934/

Log:	Handle keyword unpacking in call

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1070,9 +1070,15 @@
                     args.append(ast.Starred(expr, ast.Load,
                                             expr_node.get_lineno(),
                                             expr_node.get_column()))
+                elif expr_node.type == tokens.DOUBLESTAR:
+                    # a keyword argument unpacking
+                    expr = self.handle_expr(argument.get_child(1))
+                    args.append(ast.keyword(None, expr))
                 elif argument.get_child(1).type == syms.comp_for:
+                    # the lone generator expression
                     args.append(self.handle_genexp(argument))
                 else:
+                    # a keyword argument
                     keyword_node = argument.get_child(0)
                     keyword_expr = self.handle_expr(keyword_node)
                     if isinstance(keyword_expr, ast.Lambda):

From pypy.commits at gmail.com  Sun Jun 12 14:30:52 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sun, 12 Jun 2016 11:30:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove unneeded variable and old STAR and
 DOUBLESTAR handling in call
Message-ID: <575daa5c.03a81c0a.4267e.31d9@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85113:cbb2f3f23443
Date: 2016-06-12 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/cbb2f3f23443/

Log:	Remove unneeded variable and old STAR and DOUBLESTAR handling in
	call

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1079,27 +1079,20 @@
                     args.append(self.handle_genexp(argument))
                 else:
                     # a keyword argument
-                    keyword_node = argument.get_child(0)
-                    keyword_expr = self.handle_expr(keyword_node)
+                    keyword_expr = self.handle_expr(expr_node)
                     if isinstance(keyword_expr, ast.Lambda):
                         self.error("lambda cannot contain assignment",
-                                   keyword_node)
+                                   expr_node)
                     elif not isinstance(keyword_expr, ast.Name):
                         self.error("keyword can't be an expression",
-                                   keyword_node)
+                                   expr_node)
                     keyword = keyword_expr.id
                     if keyword in used_keywords:
-                        self.error("keyword argument repeated", keyword_node)
+                        self.error("keyword argument repeated", expr_node)
                     used_keywords[keyword] = None
-                    self.check_forbidden_name(keyword, keyword_node)
+                    self.check_forbidden_name(keyword, expr_node)
                     keyword_value = self.handle_expr(argument.get_child(2))
                     keywords.append(ast.keyword(keyword, keyword_value))
-            elif argument.type == tokens.STAR:
-                variable_arg = self.handle_expr(args_node.get_child(i + 1))
-                i += 1
-            elif argument.type == tokens.DOUBLESTAR:
-                keywords_arg = self.handle_expr(args_node.get_child(i + 1))
-                i += 1
             i += 1
         if not args:
             args = None

From pypy.commits at gmail.com  Sun Jun 12 14:37:04 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sun, 12 Jun 2016 11:37:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Missing increment for doublestars counter
 and wrong naming
Message-ID: <575dabd0.06321c0a.49cb7.401a@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85114:6c4ca5852ddd
Date: 2016-06-12 20:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6c4ca5852ddd/

Log:	Missing increment for doublestars counter and wrong naming

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1062,7 +1062,7 @@
                     args.append(self.handle_expr(expr_node))
                 elif expr_node.type == tokens.STAR:
                     # an iterable argument unpacking
-                    if ndoublestars:
+                    if doublestars_count:
                         self.error("iterable argument unpacking follows "
                                    "keyword argument unpacking",
                                    expr_node)
@@ -1074,6 +1074,7 @@
                     # a keyword argument unpacking
                     expr = self.handle_expr(argument.get_child(1))
                     args.append(ast.keyword(None, expr))
+                    doublestars_count += 1
                 elif argument.get_child(1).type == syms.comp_for:
                     # the lone generator expression
                     args.append(self.handle_genexp(argument))

From pypy.commits at gmail.com  Sun Jun 12 15:14:13 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 12 Jun 2016 12:14:13 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: Backed out changeset
 dc234c6c4b34 (didn't work)
Message-ID: <575db485.073f1c0a.88c73.3f68@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85115:77ee6d7d12af
Date: 2016-06-12 20:11 +0100
http://bitbucket.org/pypy/pypy/changeset/77ee6d7d12af/

Log:	Backed out changeset dc234c6c4b34 (didn't work)

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1183,7 +1183,6 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
-        cls.w_posix = cls.space.appexec([], GET_POSIX)
         if cls.runappdirect:
             # Can't change encoding
             try:
@@ -1202,22 +1201,25 @@
 
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
+        import posix
         try:
-            self.posix.stat(u"ą")
+            posix.stat(u"ą")
         except OSError:
             pass
 
     def test_open_unicode(self):
         # Ensure passing unicode doesn't raise UnicodeEncodeError
+        import posix
         try:
-            self.posix.open(u"ą", self.posix.O_WRONLY)
+            posix.open(u"ą", posix.O_WRONLY)
         except OSError:
             pass
 
     def test_remove_unicode(self):
         # See 2 above ;)
+        import posix
         try:
-            self.posix.remove(u"ą")
+            posix.remove(u"ą")
         except OSError:
             pass
 

From pypy.commits at gmail.com  Mon Jun 13 03:44:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 00:44:38 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Breakpoints
Message-ID: <575e6466.4f941c0a.d84c7.7353@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85117:c146f7200672
Date: 2016-06-13 09:45 +0200
http://bitbucket.org/pypy/pypy/changeset/c146f7200672/

Log:	Breakpoints

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -7,14 +7,14 @@
 from rpython.rtyper.annlowlevel import llhelper, hlstr
 
 
-def stop_point(n):
+def stop_point():
     """Indicates a point in the execution of the RPython program where
     the reverse-debugger can stop.  When reverse-debugging, we see
     the "time" as the index of the stop-point that happened.
     """
     if we_are_translated():
         if fetch_translated_config().translation.reverse_debugger:
-            llop.revdb_stop_point(lltype.Void, n)
+            llop.revdb_stop_point(lltype.Void)
 
 def register_debug_command(command, lambda_func):
     """Register the extra RPython-implemented debug command."""
@@ -25,18 +25,23 @@
 
 def current_time():
     """For RPython debug commands: returns the current time."""
-    return llop.revdb_get_value(lltype.Signed, 'c')
+    return llop.revdb_get_value(lltype.SignedLongLong, 'c')
+
+def current_break_time():
+    """Returns the time configured for the next break.  When going forward,
+    this is the target time at which we'll stop going forward."""
+    return llop.revdb_get_value(lltype.SignedLongLong, 'b')
 
 def most_recent_fork():
     """For RPython debug commands: returns the time of the most
     recent fork.  Going back to that time is fast; going back to a time
     just before is slow."""
-    return llop.revdb_get_value(lltype.Signed, 'm')
+    return llop.revdb_get_value(lltype.SignedLongLong, 'f')
 
 def total_time():
     """For RPython debug commands: returns the total time (measured
     as the total number of stop-points)."""
-    return llop.revdb_get_value(lltype.Signed, 't')
+    return llop.revdb_get_value(lltype.SignedLongLong, 't')
 
 @specialize.arg(1)
 def go_forward(time_delta, callback, arg_string):
@@ -89,8 +94,8 @@
             try:
                 cmds = t.revdb_commands
             except AttributeError:
-                cmds = t.revdb_commands = {}
-            cmds[command] = func
+                cmds = t.revdb_commands = []
+            cmds.append((command, func))
             s_func = self.bookkeeper.immutablevalue(func)
             self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
                                              s_func, [annmodel.s_Str0])
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -569,6 +569,7 @@
     'revdb_send_output':    LLOp(),
     'revdb_change_time':    LLOp(),
     'revdb_get_value':      LLOp(sideeffects=False),
+    'revdb_set_value':      LLOp(),
     'revdb_identityhash':   LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -23,7 +23,7 @@
     FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void))
 
     bk = db.translator.annotator.bookkeeper
-    cmds = getattr(db.translator, 'revdb_commands', {}).items()
+    cmds = getattr(db.translator, 'revdb_commands', [])
 
     array_names = lltype.malloc(rffi.CArray(rffi.CCHARP), len(cmds) + 1,
                                 flavor='raw', immortal=True, zero=True)
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -151,15 +151,17 @@
     /* Boehm only */
     if (obj->h_hash == 0) {
         Signed h;
+        if (flag_io_disabled) {
+            /* This is when running debug commands.  Don't cache the
+               hash on the object at all. */
+            return ~((Signed)obj);
+        }
         /* When recording, we get the hash the normal way from the
            pointer casted to an int, and record that.  When replaying,
            we read it from the record.  In both cases, we cache the
            hash in the object, so that we record/replay only once per
            object. */
-        if (flag_io_disabled)
-            h = ~((Signed)obj);
-        else
-            RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h);
+        RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h);
         assert(h != 0);
         obj->h_hash = h;
     }
@@ -297,7 +299,7 @@
     }
     fprintf(stderr, "%s", RDB_SIGNATURE);
     while ((read_all(input, 1), input[0] != 0))
-        fwrite(input, 1, 1, stderr);
+        fputc(input[0], stderr);
 
     read_all(&h, sizeof(h));
     if (h.version != RDB_VERSION) {
@@ -743,18 +745,12 @@
     cmd_go(target_time, NULL, NULL);
 }
 
-static void act_info_fork(char *p)
-{
-    printf("most_recent_fork=%llu\n", (unsigned long long)most_recent_fork);
-}
-
 static void act_info(char *p)
 {
-    static struct action_s actions_info[] = {
-        { "fork", act_info_fork },
-        { NULL }
-    };
-    process_input(p, "category", 0, actions_info);
+    char cmd = *p;
+    if (cmd == 0)
+        cmd = '?';
+    printf("info %c=%lld\n", cmd, (long long)rpy_reverse_db_get_value(cmd));
 }
 
 static void act_nop(char *p)
@@ -775,9 +771,9 @@
 static void run_debug_process(void)
 {
     static struct action_s actions_1[] = {
-        { "go", act_go },
         { "info", act_info },
         { "quit", act_quit },
+        { "__go", act_go },
         { "__forward", act_forward },
         { "", act_nop },
         { NULL }
@@ -801,7 +797,7 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_break(long stop_point)
+void rpy_reverse_db_stop_point(void)
 {
     if (process_kind == PK_MAIN_PROCESS) {
         make_new_frozen_process();
@@ -819,7 +815,7 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_change_time(char mode, Signed time,
+void rpy_reverse_db_change_time(char mode, long long time,
                                 void callback(RPyString *), RPyString *arg)
 {
     switch (mode) {
@@ -844,15 +840,17 @@
 }
 
 RPY_EXTERN
-Signed rpy_reverse_db_get_value(char value_id)
+long long rpy_reverse_db_get_value(char value_id)
 {
     switch (value_id) {
     case 'c':       /* current_time() */
         return rpy_revdb.stop_point_seen;
-    case 'm':       /* most_recent_fork() */
+    case 'f':       /* most_recent_fork() */
         return most_recent_fork;
     case 't':       /* total_time() */
         return total_stop_points;
+    case 'b':
+        return rpy_revdb.stop_point_break;
     default:
         return -1;
     }
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -67,9 +67,9 @@
 #define RPY_REVDB_EMIT_VOID(normal_code)                                \
     if (!RPY_RDB_REPLAY) { normal_code } else { }
 
-#define OP_REVDB_STOP_POINT(stop_point, r)                              \
+#define OP_REVDB_STOP_POINT(r)                                          \
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
-        rpy_reverse_db_break(stop_point)
+        rpy_reverse_db_stop_point()
 
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
@@ -86,13 +86,13 @@
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
                                       const char *file, int line);
-RPY_EXTERN void rpy_reverse_db_break(long stop_point);
+RPY_EXTERN void rpy_reverse_db_stop_point(void);
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
-RPY_EXTERN void rpy_reverse_db_change_time(char mode, Signed time,
+RPY_EXTERN void rpy_reverse_db_change_time(char mode, long long time,
                                            void callback(RPyString *),
                                            RPyString *arg);
-RPY_EXTERN Signed rpy_reverse_db_get_value(char value_id);
+RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -192,7 +192,7 @@
     def setup_class(cls):
         def main(argv):
             for op in argv[1:]:
-                revdb.stop_point(42)
+                revdb.stop_point()
                 print op
             return 9
         compile(cls, main, [], backendopt=False)
@@ -204,11 +204,11 @@
         child = self.replay()
         child.expectx('stop_points=3\r\n'
                       '(3)$ ')
-        child.sendline('go 1')
+        child.sendline('__go 1')
         child.expectx('(1)$ ')
         child.sendline('')
         child.expectx('(1)$ ')
-        child.sendline('go 52')
+        child.sendline('__go 52')
         child.expectx('(3)$ ')
 
     def test_help(self):
@@ -218,12 +218,12 @@
         # ...
         child.expectx('(3)$ ')
         child.sendline('info')
-        child.expectx("bad category '', try 'help'\r\n")
+        child.expectx("info ?=-1\r\n")
 
     def test_info_fork(self):
         child = self.replay()
         child.sendline('info fork')
-        child.expectx('most_recent_fork=3\r\n')
+        child.expectx('info f=3\r\n')
 
     def test_quit(self):
         child = self.replay()
@@ -232,19 +232,19 @@
 
     def test_forward(self):
         child = self.replay()
-        child.sendline('go 1')
+        child.sendline('__go 1')
         child.expectx('(1)$ ')
         child.sendline('__forward 1')
         child.expectx('(2)$ ')
         child.sendline('__forward 1')
         child.expectx('(3)$ ')
         child.sendline('info fork')
-        child.expectx('most_recent_fork=1\r\n')
+        child.expectx('info f=1\r\n')
         child.sendline('__forward 1')
         child.expectx('At end.\r\n'
                       '(3)$ ')
         child.sendline('info fork')
-        child.expectx('most_recent_fork=3\r\n')
+        child.expectx('info f=3\r\n')
 
 
 class TestDebugCommands(InteractiveTests):
@@ -267,6 +267,9 @@
             if revdb.current_time() != revdb.total_time():
                 revdb.go_forward(1, went_fw, "zz")
         #
+        def _nothing(arg):
+            pass
+        #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
             if cmdline == 'oops':
@@ -287,13 +290,22 @@
                 revdb.go_forward(1, went_fw, "xx")
             if cmdline == 'change-time':
                 revdb.jump_in_time(2, changed_time, "xyzzy")
+            if cmdline == 'set-break-after-0':
+                dbstate.break_after = 0
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
+        class DBState:
+            break_after = -1
+        dbstate = DBState()
+        #
         def main(argv):
             revdb.register_debug_command('r', lambda_blip)
-            for op in argv[1:]:
-                revdb.stop_point(42)
+            for i, op in enumerate(argv[1:]):
+                revdb.stop_point()
+                if i == dbstate.break_after:
+                    revdb.send_output('breakpoint!\n')
+                    revdb.go_forward(1, _nothing, "")
                 print op
             return 9
         compile(cls, main, [], backendopt=False)
@@ -318,7 +330,7 @@
     def test_interaction_with_forward(self):
         child = self.replay()
         child.expectx('(3)$ ')
-        child.sendline('go 1')
+        child.sendline('__go 1')
         child.expectx('(1)$ ')
         child.sendline('r oops')
         child.expectx('<<>>\r\n')
@@ -347,7 +359,7 @@
     def test_get_value(self):
         child = self.replay()
         child.expectx('(3)$ ')
-        child.sendline('go 2')
+        child.sendline('__go 2')
         child.expectx('(2)$ ')
         child.sendline('r get-value')
         child.expectx('<<>>\r\n'
@@ -358,7 +370,7 @@
     def test_go_fw(self):
         child = self.replay()
         child.expectx('(3)$ ')
-        child.sendline('go 1')
+        child.sendline('__go 1')
         child.expectx('(1)$ ')
         child.sendline('r go-fw')
         child.expectx('<<>>\r\n'
@@ -375,3 +387,14 @@
                       'changed-time xyzzy -> 2\r\n'
                       'went-fw zz -> 3\r\n'
                       '(3)$ ')
+
+    def test_dynamic_breakpoint(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('__go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r set-break-after-0')
+        child.expectx('(1)$ ')
+        child.sendline('__forward 5')
+        child.expectx('breakpoint!\r\n'
+                      '(2)$ ')

From pypy.commits at gmail.com  Mon Jun 13 03:44:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 00:44:36 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Rename the src directory
Message-ID: <575e6464.073f1c0a.88c73.0fac@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85116:eff7204f9b22
Date: 2016-06-13 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/eff7204f9b22/

Log:	Rename the src directory

diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c
--- a/rpython/translator/c/src/entrypoint.c
+++ b/rpython/translator/c/src/entrypoint.c
@@ -38,7 +38,7 @@
 #endif
 
 #ifdef RPY_REVERSE_DEBUGGER
-# include 
+# include 
 #endif
 
 RPY_EXPORTED
diff --git a/rpython/translator/c/src/g_include.h b/rpython/translator/c/src/g_include.h
--- a/rpython/translator/c/src/g_include.h
+++ b/rpython/translator/c/src/g_include.h
@@ -52,5 +52,5 @@
 #endif
 
 #ifdef RPY_REVERSE_DEBUGGER
-#include "rdb-src/revdb_include.h"
+#include "src-revdb/revdb_include.h"
 #endif
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -5,7 +5,7 @@
 
 
 def extra_files():
-    srcdir = py.path.local(__file__).join('..', 'rdb-src')
+    srcdir = py.path.local(__file__).join('..', 'src-revdb')
     return [
         srcdir / 'revdb.c',
     ]
diff --git a/rpython/translator/revdb/rdb-src/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
rename from rpython/translator/revdb/rdb-src/revdb.c
rename to rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/rdb-src/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -12,7 +12,7 @@
 #include "forwarddecl.h"
 #include "preimpl.h"
 #include "src/rtyper.h"
-#include "rdb-src/revdb_include.h"
+#include "src-revdb/revdb_include.h"
 
 #define RDB_SIGNATURE   "RevDB:"
 #define RDB_VERSION     0x00FF0001
diff --git a/rpython/translator/revdb/rdb-src/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
rename from rpython/translator/revdb/rdb-src/revdb_include.h
rename to rpython/translator/revdb/src-revdb/revdb_include.h

From pypy.commits at gmail.com  Mon Jun 13 05:14:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 02:14:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: Support 'llhelper(F,
 multiple_functions)', as long as they all have the
Message-ID: <575e7984.49921c0a.1e1f8.3ca7@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85118:6709ea850047
Date: 2016-06-13 10:18 +0200
http://bitbucket.org/pypy/pypy/changeset/6709ea850047/

Log:	Support 'llhelper(F, multiple_functions)', as long as they all have
	the right signature.

diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -348,19 +348,30 @@
     _about_ = llhelper
 
     def compute_result_annotation(self, s_F, s_callable):
+        from rpython.annotator.description import FunctionDesc
         assert s_F.is_constant()
-        assert s_callable.is_constant()
+        assert isinstance(s_callable, annmodel.SomePBC)
         F = s_F.const
         FUNC = F.TO
         args_s = [lltype_to_annotation(T) for T in FUNC.ARGS]
-        key = (llhelper, s_callable.const)
-        s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
-        assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
+        for desc in s_callable.descriptions:
+            assert isinstance(desc, FunctionDesc)
+            assert desc.pyobj is not None
+            if s_callable.is_constant():
+                assert s_callable.const is desc.pyobj
+            key = (llhelper, desc.pyobj)
+            s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
+            assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
         return SomePtr(F)
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
-        return hop.args_r[1].get_unique_llfn()
+        if hop.args_s[1].is_constant():
+            return hop.args_r[1].get_unique_llfn()
+        else:
+            F = hop.args_s[0].const
+            assert hop.args_r[1].lowleveltype == F
+            return hop.inputarg(hop.args_r[1], 1)
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -462,6 +462,30 @@
     res = interpret(h, [8, 5, 2])
     assert res == 99
 
+def test_llhelper_multiple_functions():
+    S = GcStruct('S', ('x', Signed), ('y', Signed))
+    def f(s):
+        return s.x - s.y
+    def g(s):
+        return s.x + s.y
+
+    F = Ptr(FuncType([Ptr(S)], Signed))
+
+    myfuncs = [f, g]
+
+    def h(x, y, z):
+        s = malloc(S)
+        s.x = x
+        s.y = y
+        fptr = llhelper(F, myfuncs[z])
+        assert typeOf(fptr) == F
+        return fptr(s)
+
+    res = interpret(h, [80, 5, 0])
+    assert res == 75
+    res = interpret(h, [80, 5, 1])
+    assert res == 85
+
 
 def test_cast_instance_to_base_ptr():
     class A:

From pypy.commits at gmail.com  Mon Jun 13 05:14:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 02:14:46 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Record the creation time of
 objects
Message-ID: <575e7986.c72d1c0a.4988f.ffff81bb@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85119:55db2cb66764
Date: 2016-06-13 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/55db2cb66764/

Log:	Record the creation time of objects

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -10,7 +10,6 @@
 class BoehmGCTransformer(GCTransformer):
     malloc_zero_filled = True
     FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
-    HDR = lltype.Struct("header", ("hash", lltype.Signed))
 
     def __init__(self, translator, inline=False):
         super(BoehmGCTransformer, self).__init__(translator, inline=inline)
@@ -28,6 +27,10 @@
         ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
         ll_malloc_varsize = mh.ll_malloc_varsize
 
+        fields = [("hash", lltype.Signed)]
+        if translator.config.translation.reverse_debugger:
+            fields.append(("ctime", lltype.SignedLongLong))
+        self.HDR = lltype.Struct("header", *fields)
         HDRPTR = lltype.Ptr(self.HDR)
 
         if self.translator:
@@ -167,7 +170,7 @@
         hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result)
 
     def gcheader_initdata(self, obj):
-        hdr = lltype.malloc(self.HDR, immortal=True)
+        hdr = lltype.malloc(self.HDR, immortal=True, zero=True)
         hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
         return hdr._obj
 
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -43,6 +43,14 @@
     as the total number of stop-points)."""
     return llop.revdb_get_value(lltype.SignedLongLong, 't')
 
+ at specialize.argtype(0)
+def creation_time_of(x):
+    """Returns the time at which the object 'x' was created.
+    More precisely, returns t such that object 'x' was created when
+    current_time()==t; this means that the object exists from time t+1.
+    """
+    return llop.revdb_creation_time_of(lltype.SignedLongLong, x)
+
 @specialize.arg(1)
 def go_forward(time_delta, callback, arg_string):
     """For RPython debug commands: tells that after this function finishes,
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -571,6 +571,7 @@
     'revdb_get_value':      LLOp(sideeffects=False),
     'revdb_set_value':      LLOp(),
     'revdb_identityhash':   LLOp(),
+    'revdb_creation_time_of': LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -578,13 +578,22 @@
                                      self.expr(op.args[0]),
                                      self.expr(op.args[1]))
 
+    def _op_boehm_malloc(self, op, is_atomic):
+        expr_result = self.expr(op.result)
+        res = 'OP_BOEHM_ZERO_MALLOC(%s, %s, void*, %d, 0);' % (
+            self.expr(op.args[0]),
+            expr_result,
+            is_atomic)
+        if self.db.reverse_debugger:
+            from rpython.translator.revdb import revdb_genc
+            res += revdb_genc.record_malloc_ctime(expr_result)
+        return res
+
     def OP_BOEHM_MALLOC(self, op):
-        return 'OP_BOEHM_ZERO_MALLOC(%s, %s, void*, 0, 0);' % (self.expr(op.args[0]),
-                                                               self.expr(op.result))
+        return self._op_boehm_malloc(op, 0)
 
     def OP_BOEHM_MALLOC_ATOMIC(self, op):
-        return 'OP_BOEHM_ZERO_MALLOC(%s, %s, void*, 1, 0);' % (self.expr(op.args[0]),
-                                                               self.expr(op.result))
+        return self._op_boehm_malloc(op, 1)
 
     def OP_BOEHM_REGISTER_FINALIZER(self, op):
         return 'GC_REGISTER_FINALIZER(%s, (GC_finalization_proc)%s, NULL, NULL, NULL);' \
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -18,6 +18,9 @@
         return emit_void(normal_code)
     return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value)
 
+def record_malloc_ctime(expr):
+    return ' RPY_REVDB_REC_CTIME(%s);' % (expr,)
+
 
 def prepare_database(db):
     FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void))
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -67,6 +67,11 @@
 #define RPY_REVDB_EMIT_VOID(normal_code)                                \
     if (!RPY_RDB_REPLAY) { normal_code } else { }
 
+#define RPY_REVDB_REC_CTIME(expr)                                       \
+    if (expr)                                                           \
+        ((struct pypy_header0 *)expr)->h_ctime = rpy_revdb.stop_point_seen
+
+
 #define OP_REVDB_STOP_POINT(r)                                          \
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
         rpy_reverse_db_stop_point()
@@ -83,6 +88,9 @@
 #define OP_REVDB_IDENTITYHASH(obj, r)                                   \
     r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
 
+#define OP_REVDB_CREATION_TIME_OF(x, r) \
+    r = ((struct pypy_header0 *)x)->h_ctime
+
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
                                       const char *file, int line);
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -191,12 +191,16 @@
 
     def setup_class(cls):
         def main(argv):
+            lst = [argv[0], 'prebuilt']
             for op in argv[1:]:
                 revdb.stop_point()
                 print op
+                lst.append(op + '??')   # create a new string here
+            for x in lst:
+                print revdb.creation_time_of(x)
             return 9
         compile(cls, main, [], backendopt=False)
-        assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
+        assert run(cls, 'abc d ef') == 'abc\nd\nef\n0\n0\n1\n2\n3\n'
         rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef'])
         assert rdb.number_of_stop_points() == 3
 

From pypy.commits at gmail.com  Mon Jun 13 05:18:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 02:18:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix for
 module.cpyext.test.test_ztranslation
Message-ID: <575e7a7b.50991c0a.f1696.ffffbd20@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85120:30a77e18d1db
Date: 2016-06-13 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/30a77e18d1db/

Log:	fix for module.cpyext.test.test_ztranslation

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -417,11 +417,11 @@
 class FakeModule(W_Root):
     def __init__(self):
         self.w_dict = w_some_obj()
-
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
+FakeObjSpace.sys.dlopenflags = 123
 FakeObjSpace.builtin = FakeModule()

From pypy.commits at gmail.com  Mon Jun 13 05:52:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 02:52:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a test checking the
 conversion_table() function and the
Message-ID: <575e826d.86c71c0a.f6f7.4c1c@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85121:4e4fa6046959
Date: 2016-06-13 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4e4fa6046959/

Log:	Add a test checking the conversion_table() function and the
	pointer_table array. Add 'immutable' hints on these arrays.

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -414,7 +414,7 @@
         if self.s_pbc.can_be_None:
             self.descriptions.insert(0, None)
         POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
-                              hints={'nolength': True})
+                              hints={'nolength': True, 'immutable': True})
         pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
                                immortal=True)
         for i, desc in enumerate(self.descriptions):
@@ -564,7 +564,7 @@
     if r_to in r_from._conversion_tables:
         return r_from._conversion_tables[r_to]
     else:
-        t = malloc(Array(Char, hints={'nolength': True}),
+        t = malloc(Array(Char, hints={'nolength': True, 'immutable': True}),
                    len(r_from.descriptions), immortal=True)
         l = []
         for i, d in enumerate(r_from.descriptions):
@@ -577,7 +577,7 @@
         if l == range(len(r_from.descriptions)):
             r = None
         else:
-            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
+            r = inputconst(typeOf(t), t)
         r_from._conversion_tables[r_to] = r
         return r
 
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -2012,6 +2012,36 @@
         e = py.test.raises(ValueError, self.interpret, f, [3])
         assert str(e.value).startswith(r"exit case '\xff' not found")
 
+    @py.test.mark.parametrize('limit', [3, 5])
+    def test_conversion_table(self, limit):
+        # with limit==3, check conversion from Char to Ptr(Func).
+        # with limit>3, check conversion from Char to Char.
+        def f1():
+            return 111
+        def f2():
+            return 222
+        def f3():
+            return 333
+        def g(n):
+            if n & 1:
+                return f1
+            else:
+                return f2
+        def f(n):
+            x = g(n)    # can be f1 or f2
+            if n > 10:
+                x = f3  # now can be f1 or f2 or f3
+            return x()
+
+        from rpython.config.translationoption import get_combined_translation_config
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.withsmallfuncsets = limit
+        assert self.interpret(f, [3]) == 111
+        assert self.interpret(f, [4]) == 222
+        assert self.interpret(f, [13]) == 333
+        assert self.interpret(f, [14]) == 333
+
+
 def test_smallfuncsets_basic():
     from rpython.translator.translator import TranslationContext, graphof
     from rpython.config.translationoption import get_combined_translation_config

From pypy.commits at gmail.com  Mon Jun 13 06:04:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 03:04:16 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge default
Message-ID: <575e8520.56311c0a.1104b.5021@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85122:ea07e83296cd
Date: 2016-06-13 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ea07e83296cd/

Log:	hg merge default

diff too long, truncating to 2000 out of 4843 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,17 +104,20 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas Stührk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -140,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +175,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,11 +210,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -228,7 +230,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -270,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +297,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -465,6 +463,7 @@
             else:
                 def __nonzero__(self):
                     return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -76,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -110,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -120,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -153,35 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
-
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
 
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -13,17 +13,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,9 +63,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -74,31 +74,34 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Edd Barrett
   Andreas Stührk
-  Edd Barrett
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -110,7 +113,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -126,11 +128,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -141,9 +145,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -153,8 +157,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -178,11 +180,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -198,7 +200,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -240,8 +241,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -265,9 +267,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
    release-5.0.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
    whatsnew-4.0.1.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,15 +53,17 @@
 immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated.
 
 
-Numpy improvements
-------------------
+NumPy rebooted
+--------------
 
-The numpy is rapidly progressing in pypy, so feel free to come to IRC and
-ask for proposed topic. A not necesarilly up-to-date `list of topics`_
-is also available.
+Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
+Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+test suite. We could use help analyzing the failures and fixing them either
+as patches to upstream NumPy, or as fixes to PyPy.
 
-.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt
-
+We also are looking for help in how to hijack NumPy dtype conversion and
+ufunc calls to allow the JIT to make them fast, using our internal _numpypy
+module.
 
 Improving the jitviewer
 ------------------------
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
--- a/pypy/doc/release-pypy2.7-v5.3.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -2,15 +2,18 @@
 PyPy2.7 v5.3
 ============
 
-We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1. 
-This release includes further improvements for the CAPI compatibility layer
-which we call cpyext. In addtion to complete support for lxml, we now pass
-most (more than 90%) of the upstream numpy test suite, and much of SciPy is
-supported as well.
+We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after
+`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3
+compatibility. This new PyPy2.7 release includes further improvements for the
+CAPI compatibility layer which we call cpyext. In addtion to complete support
+for lxml, we now pass most (more than 90%) of the upstream numpy test suite,
+and much of SciPy is supported as well.
 
-We also improved the speed of ... and ...
+We updated cffi_ to version 1.7 (small changes, documented here_).
 
-We updated cffi_ to ...
+.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html
+.. _cffi: https://cffi.readthedocs.org
+.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html
 
 You can download the PyPy2.7 v5.3 release here:
 
@@ -29,10 +32,6 @@
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
-.. _`numpy`: https://bitbucket.org/pypy/numpy
-.. _cffi: https://cffi.readthedocs.org
-.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html
-.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html
 
 What is PyPy?
 =============
@@ -41,13 +40,13 @@
 CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
 due to its integrated tracing JIT compiler.
 
-We also welcome developers of other
-`dynamic languages`_ to see what RPython can do for them.
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
 
 This release supports: 
 
   * **x86** machines on most common operating systems
-    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
   
   * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
   
@@ -65,6 +64,7 @@
 
   * Merge a major expansion of the C-API support in cpyext, here are some of
     the highlights:
+
       - allow c-snippet tests to be run with -A so we can verify we are compatible
       - fix many edge cases exposed by fixing tests to run with -A
       - issequence() logic matches cpython
@@ -72,7 +72,7 @@
       - add prelminary support for PyDateTime_*
       - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
         PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-      - PyAnySet_CheckExact, PyUnicode_Concat
+        PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo
       - improve support for PyGILState_Ensure, PyGILState_Release, and thread
         primitives, also find a case where CPython will allow thread creation
         before PyEval_InitThreads is run, dissallow on PyPy 
@@ -80,6 +80,10 @@
       - rewrite slot assignment for typeobjects
       - improve tracking of PyObject to rpython object mapping
       - support tp_as_{number, sequence, mapping, buffer} slots
+      - support ByteArrayObject via the new resizable_list_supporting_raw_ptr
+      - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's
+      - fix the signature of PyUFunc_FromFuncAndDataAndSignature
+      - implement many PyWhatever_FOO() as a macro taking a `void *`
 
   * CPyExt tweak: instead of "GIL not held when a CPython C extension module
     calls PyXxx", we now silently acquire/release the GIL.  Helps with
@@ -93,8 +97,42 @@
   * Generalize cpyext old-style buffers to more than just str/buffer, add
     support for mmap
 
+  * Support command line -v to trace import statements
+
+  * Add rposix functions for PyPy3.3 support
+
+  * Give super an __init__ and a simple __new__ for CPython compatibility
+
+  * Revive traceviewer, a tool to use pygame to view traces
+
 * Bug Fixes
 
+  * Fix issue #2277: only special-case two exact lists in zip(), not list
+    subclasses, because an overridden __iter__() should be called (probably)
+
+  * Fix issue #2226: Another tweak in the incremental GC- this should ensure
+    that progress in the major GC occurs quickly enough in all cases.
+
+  * Clarify and refactor documentation on http://doc.pypy.org
+
+  * Use "must be unicode, not %T" in unicodedata TypeErrors.
+
+  * Manually reset sys.settrace() and sys.setprofile() when we're done running.
+    This is not exactly what CPython does, but if we get an exception, unlike
+    CPython, we call functions from the 'traceback' module, and these would
+    call more the trace/profile function.  That's unexpected and can lead
+    to more crashes at this point.
+
+  * Use the appropriate tp_dealloc on a subclass of a builtin type, and call
+    tp_new for a python-sublcass of a C-API type
+
+  * Fix for issue #2285 - rare vmprof segfaults on OS/X
+
+  * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc
+
+  * Fix issue #2311 - grab the `__future__` flags imported in the main script, in
+    `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console
+
   * Issues reported with our previous release were resolved_ after reports from users on
     our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
     #pypy
@@ -103,6 +141,9 @@
 
   * Implement ufunc.outer on numpypy
 
+  * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_
+    accordingly)
+
 * Performance improvements:
 
   * Use bitstrings to compress lists of descriptors that are attached to an
@@ -114,11 +155,20 @@
     can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
     negative.
 
+  * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())`
+
+  * Use the __builtin_add_overflow built-ins if they are available
+
+  * Rework the way registers are moved/spilled in before_call()
 
 * Internal refactorings:
 
+  * Refactor code to better support Python3-compatible syntax
+
+  * Document and refactor OperationError -> oefmt
+
   * Reduce the size of generated C sources during translation by 
-    refactoring function declarations
+    eliminating many many unused struct declarations (Issue #2281)
 
   * Remove a number of translation-time options that were not tested and
     never used. Also fix a performance bug in the method cache
@@ -126,10 +176,14 @@
   * Reduce the size of generated code by using the same function objects in
     all generated subclasses
 
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
+
   * Compile c snippets with -Werror, and fix warnings it exposed
 
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
 .. _Numpy: https://bitbucket.org/pypy/numpy
+.. _`the repo`: https://bitbucket.org/pypy/numpy
 
 Please update, and continue to help us make PyPy better.
 
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -73,6 +73,8 @@
     'Richard Lancaster':['richardlancaster'],
     'William Leslie':['William ML Leslie'],
     'Spenser Bauman':['Spenser Andrew Bauman'],
+    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
+    'timo':['timo at eistee.fritz.box'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,145 +1,33 @@
 =========================
-What's new in PyPy 5.1+
+What's new in PyPy2.7 5.3+
 =========================
 
-.. this is a revision shortly after release-5.1
-.. startrev: aa60332382a1
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
 
-.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
 
-.. branch: gcheader-decl
+.. branch: fix-gen-dfa
 
-Reduce the size of generated C sources.
+Resolves an issue with the generator script to build the dfa for Python syntax.
 
+.. branch: z196-support
 
-.. branch: remove-objspace-options
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
 
-Remove a number of options from the build process that were never tested and
-never set. Fix a performance bug in the method cache.
+.. branch: s390x-5.3-catchup
 
-.. branch: bitstring
+Implement the backend related changes for s390x.
 
-JIT: use bitstrings to compress the lists of read or written descrs
-that we attach to EffectInfo.  Fixes a problem we had in
-remove-objspace-options.
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
 
-.. branch: cpyext-for-merge
+.. branch: testing-cleanup
 
-Update cpyext C-API support After this branch, we are almost able to support 
-upstream numpy via cpyext, so we created (yet another) fork of numpy at 
-github.com/pypy/numpy with the needed changes. Among the significant changes 
-to cpyext:
-  - allow c-snippet tests to be run with -A so we can verify we are compatible
-  - fix many edge cases exposed by fixing tests to run with -A
-  - issequence() logic matches cpython
-  - make PyStringObject and PyUnicodeObject field names compatible with cpython
-  - add prelminary support for PyDateTime_*
-  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
-    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-  - PyAnySet_CheckExact, PyUnicode_Concat
-  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
-    primitives, also find a case where CPython will allow thread creation
-    before PyEval_InitThreads is run, dissallow on PyPy 
-  - create a PyObject-specific list strategy
-  - rewrite slot assignment for typeobjects
-  - improve tracking of PyObject to rpython object mapping
-  - support tp_as_{number, sequence, mapping, buffer} slots
+Simplify handling of interp-level tests and make it more forward-
+compatible.
 
-(makes the pypy-c bigger; this was fixed subsequently by the
-share-cpyext-cpython-api branch)
-
-.. branch: share-mapdict-methods-2
-
-Reduce generated code for subclasses by using the same function objects in all
-generated subclasses.
-
-.. branch: share-cpyext-cpython-api
-
-.. branch: cpyext-auto-gil
-
-CPyExt tweak: instead of "GIL not held when a CPython C extension module
-calls PyXxx", we now silently acquire/release the GIL.  Helps with
-CPython C extension modules that call some PyXxx() functions without
-holding the GIL (arguably, they are theorically buggy).
-
-.. branch: cpyext-test-A
-
-Get the cpyext tests to pass with "-A" (i.e. when tested directly with
-CPython).
-
-.. branch: oefmt
-
-.. branch: cpyext-werror
-
-Compile c snippets with -Werror in cpyext
-
-.. branch: gc-del-3
-
-Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
-It is a more flexible way to make RPython finalizers.
-
-.. branch: unpacking-cpython-shortcut
-
-.. branch: cleanups
-
-.. branch: cpyext-more-slots
-
-.. branch: use-gc-del-3
-
-Use the new rgc.FinalizerQueue mechanism to clean up the handling of
-``__del__`` methods.  Fixes notably issue #2287.  (All RPython
-subclasses of W_Root need to use FinalizerQueue now.)
-
-.. branch: ufunc-outer
-
-Implement ufunc.outer on numpypy
-
-.. branch: verbose-imports
-
-Support ``pypy -v``: verbose imports.  It does not log as much as
-cpython, but it should be enough to help when debugging package layout
-problems.
-
-.. branch: cpyext-macros-cast
-
-Fix some warnings when compiling CPython C extension modules
-
-.. branch: syntax_fix
-
-.. branch: remove-raisingops
-
-Remove most of the _ovf, _zer and _val operations from RPython.  Kills
-quite some code internally, and allows the JIT to do better
-optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
-can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
-negative.
-
-.. branch: cpyext-old-buffers
-
-Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
-
-.. branch: numpy-includes
-
-Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
-This allows building upstream numpy and scipy in pypy via cpyext
-
-.. branch: traceviewer-common-merge-point-formats
-
-Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
-
-.. branch: cpyext-pickle
-
-Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
-at cpyext import time
-
-.. branch: nonmovable-list
-
-Add a way to ask "give me a raw pointer to this list's
-items".  Only for resizable lists of primitives.  Turns the GcArray
-nonmovable, possibly making a copy of it first.
-
-.. branch: cpyext-ext
-
-Finish the work already partially merged in cpyext-for-merge. Adds support
-for ByteArrayObject using the nonmovable-list, which also enables
-buffer(bytearray()) 
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -0,0 +1,145 @@
+=========================
+What's new in PyPy2.7 5.3
+=========================
+
+.. this is a revision shortly after release-5.1
+.. startrev: aa60332382a1
+
+.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+
+.. branch: gcheader-decl
+
+Reduce the size of generated C sources.
+
+
+.. branch: remove-objspace-options
+
+Remove a number of options from the build process that were never tested and
+never set. Fix a performance bug in the method cache.
+
+.. branch: bitstring
+
+JIT: use bitstrings to compress the lists of read or written descrs
+that we attach to EffectInfo.  Fixes a problem we had in
+remove-objspace-options.
+
+.. branch: cpyext-for-merge
+
+Update cpyext C-API support After this branch, we are almost able to support 
+upstream numpy via cpyext, so we created (yet another) fork of numpy at 
+github.com/pypy/numpy with the needed changes. Among the significant changes 
+to cpyext:
+  - allow c-snippet tests to be run with -A so we can verify we are compatible
+  - fix many edge cases exposed by fixing tests to run with -A
+  - issequence() logic matches cpython
+  - make PyStringObject and PyUnicodeObject field names compatible with cpython
+  - add prelminary support for PyDateTime_*
+  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
+    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
+  - PyAnySet_CheckExact, PyUnicode_Concat
+  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
+    primitives, also find a case where CPython will allow thread creation
+    before PyEval_InitThreads is run, dissallow on PyPy 
+  - create a PyObject-specific list strategy
+  - rewrite slot assignment for typeobjects
+  - improve tracking of PyObject to rpython object mapping
+  - support tp_as_{number, sequence, mapping, buffer} slots
+
+(makes the pypy-c bigger; this was fixed subsequently by the
+share-cpyext-cpython-api branch)
+
+.. branch: share-mapdict-methods-2
+
+Reduce generated code for subclasses by using the same function objects in all
+generated subclasses.
+
+.. branch: share-cpyext-cpython-api
+
+.. branch: cpyext-auto-gil
+
+CPyExt tweak: instead of "GIL not held when a CPython C extension module
+calls PyXxx", we now silently acquire/release the GIL.  Helps with
+CPython C extension modules that call some PyXxx() functions without
+holding the GIL (arguably, they are theorically buggy).
+
+.. branch: cpyext-test-A
+
+Get the cpyext tests to pass with "-A" (i.e. when tested directly with
+CPython).
+
+.. branch: oefmt
+
+.. branch: cpyext-werror
+
+Compile c snippets with -Werror in cpyext
+
+.. branch: gc-del-3
+
+Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
+It is a more flexible way to make RPython finalizers.
+
+.. branch: unpacking-cpython-shortcut
+
+.. branch: cleanups
+
+.. branch: cpyext-more-slots
+
+.. branch: use-gc-del-3
+
+Use the new rgc.FinalizerQueue mechanism to clean up the handling of
+``__del__`` methods.  Fixes notably issue #2287.  (All RPython
+subclasses of W_Root need to use FinalizerQueue now.)
+
+.. branch: ufunc-outer
+
+Implement ufunc.outer on numpypy
+
+.. branch: verbose-imports
+
+Support ``pypy -v``: verbose imports.  It does not log as much as
+cpython, but it should be enough to help when debugging package layout
+problems.
+
+.. branch: cpyext-macros-cast
+
+Fix some warnings when compiling CPython C extension modules
+
+.. branch: syntax_fix
+
+.. branch: remove-raisingops
+
+Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+quite some code internally, and allows the JIT to do better
+optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+negative.
+
+.. branch: cpyext-old-buffers
+
+Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
+
+.. branch: numpy-includes
+
+Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
+This allows building upstream numpy and scipy in pypy via cpyext
+
+.. branch: traceviewer-common-merge-point-formats
+
+Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
+
+.. branch: cpyext-pickle
+
+Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
+at cpyext import time
+
+.. branch: nonmovable-list
+
+Add a way to ask "give me a raw pointer to this list's
+items".  Only for resizable lists of primitives.  Turns the GcArray
+nonmovable, possibly making a copy of it first.
+
+.. branch: cpyext-ext
+
+Finish the work already partially merged in cpyext-for-merge. Adds support
+for ByteArrayObject using the nonmovable-list, which also enables
+buffer(bytearray()) 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -458,14 +458,17 @@
         decl = str(decl) + "\n"
         yield self.st, decl, 'x', (1, 2, 3, 4)
 
+    def test_closure_error(self):
         source = """if 1:
         def f(a):
             del a
             def x():
                 a
         """
-        exc = py.test.raises(SyntaxError, self.run, source).value
-        assert exc.msg == "Can't delete variable used in nested scopes: 'a'"
+        with py.test.raises(SyntaxError) as excinfo:
+            self.run(source)
+        msg = excinfo.value.msg
+        assert msg == "Can't delete variable used in nested scopes: 'a'"
 
     def test_try_except_finally(self):
         yield self.simple_test, """
@@ -879,7 +882,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -903,19 +919,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
@@ -1111,7 +1115,7 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
-        
+
     def test_list_comprehension(self):
         source = "def f(): [i for i in l]"
         source2 = "def f(): [i for i in l for j in l]"
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -205,7 +205,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    map = {"'" : (singleDFA, states_singleDFA),
+           '"' : (doubleDFA, states_doubleDFA),
            "r" : None,
            "R" : None,
            "u" : None,
@@ -257,25 +265,30 @@
     for uniPrefix in ("", "u", "U", "b", "B", ):
         for rawPrefix in ("", "r", "R"):
             prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
+            map[prefix + "'''"] = (single3DFA, states_single3DFA)
+            map[prefix + '"""'] = (double3DFA, states_double3DFA)
     return map
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append('\n')
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -298,22 +311,28 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap["\""]
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.genpytokenize import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -48,10 +48,10 @@
             return f.f_code
         assert g() is g.func_code
 
-    def test_f_trace_del(self): 
+    def test_f_trace_del(self):
         import sys
-        f = sys._getframe() 
-        del f.f_trace 
+        f = sys._getframe()
+        del f.f_trace
         assert f.f_trace is None
 
     def test_f_lineno(self):
@@ -116,7 +116,7 @@
         def f():
             assert sys._getframe().f_code.co_name == g()
         def g():
-            return sys._getframe().f_back.f_code.co_name 
+            return sys._getframe().f_back.f_code.co_name
         f()
 
     def test_f_back_virtualref(self):
@@ -233,7 +233,7 @@
     def test_trace_exc(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -298,7 +298,7 @@
     def test_trace_return_exc(self):
         import sys
         l = []
-        def trace(a,b,c): 
+        def trace(a,b,c):
             if b in ('exception', 'return'):
                 l.append((b, c))
             return trace
@@ -444,7 +444,7 @@
     def test_dont_trace_on_reraise(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -466,7 +466,7 @@
     def test_dont_trace_on_raise_with_tb(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -3,7 +3,7 @@
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
-class AppTestSlow:    
+class AppTestSlow:
     spaceconfig = dict(usemodules=['itertools'])
 
     def setup_class(cls):
@@ -64,7 +64,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('read_exc_type'),
                   space.wrap(read_exc_type_gw))
-    
+
 def _detach_helpers(space):
     space.delitem(space.builtin.w_dict,
                   space.wrap('hide_top_frame'))
@@ -92,7 +92,7 @@
         pckl = pickle.dumps(code)
         result = pickle.loads(pckl)
         assert code == result
-    
+
     def test_pickle_global_func(self):
         import new
         mod = new.module('mod')
@@ -109,7 +109,7 @@
             assert func is result
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_not_imported_module(self):
         import new
         mod = new.module('mod')
@@ -119,13 +119,13 @@
         result = pickle.loads(pckl)
         assert mod.__name__ == result.__name__
         assert mod.__dict__ == result.__dict__
-    
+
     def test_pickle_builtin_func(self):
         import pickle
         pckl = pickle.dumps(map)
         result = pickle.loads(pckl)
         assert map is result
-    
+
     def test_pickle_non_top_reachable_func(self):
         def func():
             return 42
@@ -142,7 +142,7 @@
         assert func.func_dict     == result.func_dict
         assert func.func_doc      == result.func_doc
         assert func.func_globals  == result.func_globals
-    
+
     def test_pickle_cell(self):
         def g():
             x = [42]
@@ -171,7 +171,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
         assert type(f1) is type(f2)
@@ -223,7 +223,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
     def test_frame_setstate_crash(self):
@@ -257,21 +257,21 @@
         pckl   = pickle.dumps(mod)
         result = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_moduledict(self):
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
         result   = pickle.loads(pckl)
         assert moddict is result
-    
+
     def test_pickle_bltins_module(self):
         import pickle
         mod  = __builtins__
         pckl     = pickle.dumps(mod)
         result   = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_buffer(self):
         skip("Can't pickle buffer objects on top of CPython either.  "
              "Do we really need it?")
@@ -280,14 +280,14 @@
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_complex(self):
         import pickle
         a = complex(1.23,4.567)
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_method(self):
         class myclass(object):
             def f(self):
@@ -308,7 +308,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_staticmethod(self):
         class myclass(object):
             def f():
@@ -319,7 +319,7 @@
         pckl     = pickle.dumps(method)
         result   = pickle.loads(pckl)
         assert method() == result()
-    
+
     def test_pickle_classmethod(self):
         class myclass(object):
             def f(cls):
@@ -337,7 +337,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_sequenceiter(self):
         '''
         In PyPy there is no distinction here between listiterator and
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.6.0"
+VERSION = "1.7.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -420,6 +420,14 @@
             w_result = ctype.ctitem.unpack_ptr(ctype, ptr, length)
         return w_result
 
+    def dir(self, space):
+        from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
+        ct = self.ctype
+        if isinstance(ct, W_CTypePointer):
+            ct = ct.ctitem
+        lst = ct.cdata_dir()
+        return space.newlist([space.wrap(s) for s in lst])
+
 
 class W_CDataMem(W_CData):
     """This is used only by the results of cffi.cast('int', x)
@@ -602,5 +610,6 @@
     __call__ = interp2app(W_CData.call),
     __iter__ = interp2app(W_CData.iter),
     __weakref__ = make_weakref_descr(W_CData),
+    __dir__ = interp2app(W_CData.dir),
     )
 W_CData.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -256,6 +256,9 @@
     def fget_elements(self, space): return self._fget('e')
     def fget_relements(self, space):return self._fget('R')
 
+    def cdata_dir(self):
+        return []
+
 
 W_CType.typedef = TypeDef(
     '_cffi_backend.CTypeDescr',
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -171,6 +171,12 @@
                 pass
         return W_CType.getcfield(self, attr)
 
+    def cdata_dir(self):
+        if self.size < 0:
+            return []
+        self.force_lazy_struct()
+        return self._fields_dict.keys()
+
 
 class W_CTypeStruct(W_CTypeStructOrUnion):
     kind = "struct"
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -201,6 +201,9 @@
         else:
             copy_string_to_raw(llstr(src_string), dest_data, 0, n)
     else:
+        # nowadays this case should be rare or impossible: as far as
+        # I know, all common types implementing the *writable* buffer
+        # interface now support get_raw_address()
         if src_is_ptr:
             for i in range(n):
                 dest_buf.setitem(i, src_data[i])
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.6.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -77,8 +77,8 @@
     assert repr(p) == ""
 
 def check_dir(p, expected):
-    got = set(name for name in dir(p) if not name.startswith('_'))
-    assert got == set(expected)
+    got = [name for name in dir(p) if not name.startswith('_')]
+    assert got == sorted(expected)
 
 def test_inspect_primitive_type():
     p = new_primitive_type("signed char")
@@ -3608,3 +3608,23 @@
     #
     py.test.raises(ValueError, unpack, p0, -1)
     py.test.raises(ValueError, unpack, p, -1)
+
+def test_cdata_dir():
+    BInt = new_primitive_type("int")
+    p = cast(BInt, 42)
+    check_dir(p, [])
+    p = newp(new_array_type(new_pointer_type(BInt), None), 5)
+    check_dir(p, [])
+    BStruct = new_struct_type("foo")
+    p = cast(new_pointer_type(BStruct), 0)
+    check_dir(p, [])    # opaque
+    complete_struct_or_union(BStruct, [('a2', BInt, -1),
+                                       ('a1', BInt, -1)])
+    check_dir(p, ['a1', 'a2'])   # always sorted
+    p = newp(new_pointer_type(BStruct), None)
+    check_dir(p, ['a1', 'a2'])
+    check_dir(p[0], ['a1', 'a2'])
+    pp = newp(new_pointer_type(new_pointer_type(BStruct)), p)
+    check_dir(pp, [])
+    check_dir(pp[0], ['a1', 'a2'])
+    check_dir(pp[0][0], ['a1', 'a2'])
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -671,13 +671,11 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket', 'array']}
 
     def setup_method(self, method):
-        w_HOST = space.wrap(self.HOST)
-        self.w_serv = space.appexec([w_socket, w_HOST],
+        w_HOST = self.space.wrap(self.HOST)
+        self.w_serv =self.space.appexec([w_socket, w_HOST],
             '''(_socket, HOST):
             serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
             serv.bind((HOST, 0))
@@ -687,7 +685,7 @@
 
     def teardown_method(self, method):
         if hasattr(self, 'w_serv'):
-            space.appexec([self.w_serv], '(serv): serv.close()')
+            self.space.appexec([self.w_serv], '(serv): serv.close()')
             self.w_serv = None
 
     def test_timeout(self):
@@ -803,8 +801,7 @@
 
 
 class AppTestErrno:
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -3,8 +3,9 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
+    spaceconfig = {'usemodules': ['_vmprof', 'struct']}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1')))
         cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2')))
 
@@ -17,7 +18,7 @@
         import struct, sys, gc
 
         WORD = struct.calcsize('l')
-        
+
         def count(s):
             i = 0
             count = 0
@@ -44,7 +45,7 @@
                 else:
                     raise AssertionError(ord(s[i]))
             return count
-        
+
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1512,7 +1512,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.0-alpha0"
-#define PYPY_VERSION_NUM  0x05030000
+#define PYPY_VERSION "5.3.1-alpha0"
+#define PYPY_VERSION_NUM  0x05030100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer,
+    PyVarObject, Py_buffer, size_t,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
@@ -14,14 +14,14 @@
 import pypy.module.__builtin__.operation as operation
 
 
- at cpython_api([Py_ssize_t], rffi.VOIDP)
+ at cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
     return lltype.malloc(rffi.VOIDP.TO, size,
                          flavor='raw',
                          add_memory_pressure=True)
 
- at cpython_api([rffi.VOIDP, Py_ssize_t], rffi.VOIDP)
+ at cpython_api([rffi.VOIDP, size_t], rffi.VOIDP)
 def PyObject_Realloc(space, ptr, size):
     if not lltype.cast_ptr_to_int(ptr):
         return lltype.malloc(rffi.VOIDP.TO, size,
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -143,7 +143,7 @@
 def _setup(space):
     dn = setup_directory_structure(space)
     return space.appexec([space.wrap(dn)], """
-        (dn): 
+        (dn):
             import sys
             path = list(sys.path)
             sys.path.insert(0, dn)
@@ -1017,7 +1017,7 @@
 
         cpathname = udir.join('test.pyc')
         assert not cpathname.check()
-        
+
     def test_load_source_module_importerror(self):
         # the .pyc file is created before executing the module
         space = self.space
@@ -1126,11 +1126,11 @@
                     stream.close()
 
 
-def test_PYTHONPATH_takes_precedence(space): 
+def test_PYTHONPATH_takes_precedence(space):
     if sys.platform == "win32":
         py.test.skip("unresolved issues with win32 shell quoting rules")
-    from pypy.interpreter.test.test_zpy import pypypath 
-    extrapath = udir.ensure("pythonpath", dir=1) 
+    from pypy.interpreter.test.test_zpy import pypypath
+    extrapath = udir.ensure("pythonpath", dir=1)
     extrapath.join("sched.py").write("print 42\n")
     old = os.environ.get('PYTHONPATH', None)
     oldlang = os.environ.pop('LANG', None)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -495,8 +495,8 @@
         c = array([1.e+110, 1.e-110], dtype=complex128)
         d = floor_divide(c**2, c)
         assert (d == [1.e+110, 0]).all()
-        
-        
+
+
 
     def test_basic(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -372,8 +372,8 @@
         a = np.array(data, dtype=b)
         x = pickle.loads(pickle.dumps(a))
         assert (x == a).all()
-        assert x.dtype == a.dtype 
-        
+        assert x.dtype == a.dtype
+
     def test_index(self):
         import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
@@ -1459,7 +1459,7 @@
                      "'offsets':[0,76800], "
                      "'itemsize':80000, "
                      "'aligned':True}")
-        
+
         assert dt == np.dtype(eval(str(dt)))
 
         dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1878,7 +1878,7 @@
         assert map(isnan, e) == [False, False, False, True, False]
         assert map(isinf, e) == [False, False, True, False, False]
         assert e.argmax() == 3
-        # numpy preserves value for uint16 -> cast_as_float16 -> 
+        # numpy preserves value for uint16 -> cast_as_float16 ->
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
         all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
@@ -2608,7 +2608,7 @@
         a = np.arange(6).reshape(2,3)
         i = np.dtype('int32').type(0)
         assert (a[0] == a[i]).all()
-        
+
 
     def test_ellipsis_indexing(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -200,7 +200,7 @@
         from numpy import arange, dtype
         from cPickle import loads, dumps
         import sys
-        
+
         a = arange(15).astype(object)
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, dumps, a)
@@ -211,4 +211,4 @@
         a = arange(15).astype(object).reshape((3, 5))
         b = loads(dumps(a))
         assert (a == b).all()
-        
+
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -13,14 +13,15 @@
 import sys
 import signal
 
+USEMODULES = ['binascii', 'posix', 'struct', 'time']
+if os.name != 'nt':
+    USEMODULES += ['fcntl']
+else:
+    # On windows, os.popen uses the subprocess module
+    USEMODULES += ['_rawffi', 'thread', 'signal']
+
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'struct', 'time']
-    if os.name != 'nt':
-        usemodules += ['fcntl']
-    else:
-        # On windows, os.popen uses the subprocess module
-        usemodules += ['_rawffi', 'thread', 'signal']
-    mod.space = gettestobjspace(usemodules=usemodules)
+    mod.space = gettestobjspace(usemodules=USEMODULES)
     mod.path = udir.join('posixtestfile.txt')
     mod.path.write("this is a test")
     mod.path2 = udir.join('test_posix2-')
@@ -49,9 +50,10 @@
 
 
 class AppTestPosix:
+    spaceconfig = {'usemodules': USEMODULES}
 
     def setup_class(cls):
-        cls.space = space
+        space = cls.space
         cls.w_runappdirect = space.wrap(cls.runappdirect)
         cls.w_posix = space.appexec([], GET_POSIX)
         cls.w_path = space.wrap(str(path))
@@ -1145,14 +1147,10 @@
 
 class AppTestEnvironment(object):
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name)
-        cls.w_os = space.appexec([], "(): import os; return os")
         cls.w_path = space.wrap(str(path))
 
     def test_environ(self):
-        posix = self.posix
-        os = self.os
+        import posix
         assert posix.environ['PATH']
         del posix.environ['PATH']
         def fn(): posix.environ['PATH']
@@ -1160,7 +1158,7 @@
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
-            os = self.os
+            import os
             os.unsetenv("XYZABC") #does not raise
             try:
                 os.environ["ABCABC"]
@@ -1178,8 +1176,6 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):

From pypy.commits at gmail.com  Mon Jun 13 06:04:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 03:04:18 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Don't record the reads out of
 the small funcset arrays
Message-ID: <575e8522.cf981c0a.ede76.4e27@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85123:1f9da349e92c
Date: 2016-06-13 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/1f9da349e92c/

Log:	Don't record the reads out of the small funcset arrays

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -47,7 +47,8 @@
 def creation_time_of(x):
     """Returns the time at which the object 'x' was created.
     More precisely, returns t such that object 'x' was created when
-    current_time()==t; this means that the object exists from time t+1.
+    current_time()==t; this means that the object exists at the stop
+    point number t+1, but does not exist yet at the stop point number t.
     """
     return llop.revdb_creation_time_of(lltype.SignedLongLong, x)
 
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -414,7 +414,8 @@
         if self.s_pbc.can_be_None:
             self.descriptions.insert(0, None)
         POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
-                              hints={'nolength': True, 'immutable': True})
+                              hints={'nolength': True, 'immutable': True,
+                                     'static_immutable': True})
         pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
                                immortal=True)
         for i, desc in enumerate(self.descriptions):
@@ -564,7 +565,8 @@
     if r_to in r_from._conversion_tables:
         return r_from._conversion_tables[r_to]
     else:
-        t = malloc(Array(Char, hints={'nolength': True, 'immutable': True}),
+        t = malloc(Array(Char, hints={'nolength': True, 'immutable': True,
+                                      'static_immutable': True}),
                    len(r_from.descriptions), immortal=True)
         l = []
         for i, d in enumerate(r_from.descriptions):
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -59,12 +59,15 @@
         return self.cur == len(self.buffer)
 
 
-def compile(self, entry_point, argtypes, backendopt=True):
+def compile(self, entry_point, argtypes, backendopt=True,
+            withsmallfuncsets=None):
     t = Translation(entry_point, None, gc="boehm")
     self.t = t
     t.config.translation.reverse_debugger = True
     t.config.translation.rweakref = False
     t.config.translation.lldebug0 = True
+    if withsmallfuncsets is not None:
+        t.config.translation.withsmallfuncsets = withsmallfuncsets
     if not backendopt:
         t.disable(["backendopt_lltype"])
     t.annotate()
@@ -171,6 +174,40 @@
         x = rdb.next('q'); assert x == 0      # number of stop points
         assert rdb.done()
 
+    @py.test.mark.parametrize('limit', [3, 5])
+    def test_dont_record_small_funcset_conversions(self, limit):
+        def f1():
+            return 111
+        def f2():
+            return 222
+        def f3():
+            return 333
+        def g(n):
+            if n & 1:
+                return f1
+            else:
+                return f2
+        def main(argv):
+            x = g(len(argv))    # can be f1 or f2
+            if len(argv) > 5:
+                x = f3  # now can be f1 or f2 or f3
+            print x()
+            return 9
+        self.compile(main, [], backendopt=False, withsmallfuncsets=limit)
+        for input, expected_output in [
+                ('2 3', '111\n'),
+                ('2 3 4', '222\n'),
+                ('2 3 4 5 6 7', '333\n'),
+                ]:
+            out = self.run(input)
+            assert out == expected_output
+            rdb = self.fetch_rdb([self.exename] + input.split())
+            # write() call
+            x = rdb.next(); assert x == len(out)
+            x = rdb.next('i'); assert x == 0      # errno
+            x = rdb.next('q'); assert x == 0      # number of stop points
+            assert rdb.done()
+
 
 class InteractiveTests(object):
     EOF = pexpect.EOF

From pypy.commits at gmail.com  Mon Jun 13 07:03:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 04:03:04 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: object_to_id
Message-ID: <575e92e8.4a9bc20a.b68d3.ffffadf3@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85124:6494d80c50d8
Date: 2016-06-13 13:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6494d80c50d8/

Log:	object_to_id

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -5,6 +5,8 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.annlowlevel import llhelper, hlstr
+from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
+from rpython.rtyper.rclass import OBJECTPTR
 
 
 def stop_point():
@@ -52,6 +54,15 @@
     """
     return llop.revdb_creation_time_of(lltype.SignedLongLong, x)
 
+ at specialize.argtype(0)
+def object_to_id(x):
+    return llop.revdb_object_to_id(lltype.Signed, x)
+
+ at specialize.arg(0)
+def id_to_object(Class, object_id):
+    x = llop.revdb_id_to_object(OBJECTPTR, object_id)
+    return cast_base_ptr_to_instance(Class, x)
+
 @specialize.arg(1)
 def go_forward(time_delta, callback, arg_string):
     """For RPython debug commands: tells that after this function finishes,
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -572,6 +572,8 @@
     'revdb_set_value':      LLOp(),
     'revdb_identityhash':   LLOp(),
     'revdb_creation_time_of': LLOp(sideeffects=False),
+    'revdb_object_to_id':   LLOp(sideeffects=False),
+    'revdb_id_to_object':   LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -79,7 +79,7 @@
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
 
-#define OP_REVDB_CHANGE_TIME(mode, time, callback, ll_string, r)   \
+#define OP_REVDB_CHANGE_TIME(mode, time, callback, ll_string, r)        \
     rpy_reverse_db_change_time(mode, time, callback, ll_string)
 
 #define OP_REVDB_GET_VALUE(value_id, r)                                 \
@@ -88,9 +88,15 @@
 #define OP_REVDB_IDENTITYHASH(obj, r)                                   \
     r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
 
-#define OP_REVDB_CREATION_TIME_OF(x, r) \
+#define OP_REVDB_CREATION_TIME_OF(x, r)                                 \
     r = ((struct pypy_header0 *)x)->h_ctime
 
+#define OP_REVDB_OBJECT_TO_ID(x, r)                                     \
+    r = (Signed)x
+
+#define OP_REVDB_ID_TO_OBJECT(x, r)                                     \
+    r = (void *)x
+
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
                                       const char *file, int line);
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -292,6 +292,9 @@
 
     def setup_class(cls):
         #
+        class Stuff:
+            pass
+        #
         def g(cmdline):
             if len(cmdline) > 5:
                 raise ValueError
@@ -333,6 +336,12 @@
                 revdb.jump_in_time(2, changed_time, "xyzzy")
             if cmdline == 'set-break-after-0':
                 dbstate.break_after = 0
+            if cmdline == 'print-id':
+                revdb.send_output('%d\n' % (revdb.object_to_id(dbstate.stuff),))
+            if cmdline.startswith('check-id '):
+                obj_id = int(cmdline[len('check-id '):])
+                revdb.send_output("%d\n" %
+                    int(revdb.id_to_object(Stuff, obj_id) is dbstate.stuff))
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -341,6 +350,7 @@
         dbstate = DBState()
         #
         def main(argv):
+            dbstate.stuff = Stuff()
             revdb.register_debug_command('r', lambda_blip)
             for i, op in enumerate(argv[1:]):
                 revdb.stop_point()
@@ -439,3 +449,22 @@
         child.sendline('__forward 5')
         child.expectx('breakpoint!\r\n'
                       '(2)$ ')
+
+    def test_object_to_id(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r print-id')
+        child.expect(re.escape('<<>>\r\n')
+                     + r'(-?\d+)'
+                     + re.escape('\r\n'
+                                 'blipped\r\n'
+                                 '(3)$ '))
+        object_id = child.match.group(1)
+        for at_time in [1, 2, 3]:
+            child.sendline('__go %d' % at_time)
+            child.expectx('(%d)$ ' % at_time)
+            child.sendline('r check-id ' + object_id)
+            child.expectx('<<>>\r\n' % (object_id,) +
+                          '1\r\n' +
+                          'blipped\r\n' +
+                          '(%d)$ ' % at_time)

From pypy.commits at gmail.com  Mon Jun 13 09:02:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 06:02:17 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix
Message-ID: <575eaed9.6a56c20a.b129a.5324@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85125:da8e81bf6327
Date: 2016-06-13 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/da8e81bf6327/

Log:	fix

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -224,6 +224,7 @@
 static jmp_buf jmp_buf_cancel_execution;
 static uint64_t most_recent_fork;
 static uint64_t total_stop_points;
+static uint64_t stopped_time;
 
 static void (*invoke_after_forward)(RPyString *);
 static RPyString *invoke_argument;
@@ -552,7 +553,7 @@
 
     if (process_kind == PK_DEBUG_PROCESS) {
         printf("At end.\n");
-        cmd_go(rpy_revdb.stop_point_seen, NULL, NULL);
+        cmd_go(stop_points, NULL, NULL);
         abort();   /* unreachable */
     }
 
@@ -779,12 +780,13 @@
         { NULL }
     };
     while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
+        stopped_time = rpy_revdb.stop_point_seen;
         if (invoke_after_forward != NULL) {
             execute_rpy_function(invoke_after_forward, invoke_argument);
         }
         else {
             char input[256];
-            printf("(%llu)$ ", (unsigned long long)rpy_revdb.stop_point_seen);
+            printf("(%llu)$ ", (unsigned long long)stopped_time);
             fflush(stdout);
             if (fgets(input, sizeof(input), stdin) != input) {
                 fprintf(stderr, "\n");
@@ -793,6 +795,7 @@
             }
             process_input(input, "command", 1, actions_1);
         }
+        stopped_time = 0;
     }
 }
 
@@ -825,7 +828,11 @@
             fprintf(stderr, "revdb.go_forward(): negative amount of steps\n");
             exit(1);
         }
-        rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + time;
+        if (stopped_time == 0) {
+            fprintf(stderr, "revdb.go_forward(): not from a debug command\n");
+            exit(1);
+        }
+        rpy_revdb.stop_point_break = stopped_time + time;
         invoke_after_forward = callback;
         invoke_argument = arg;
         break;
@@ -844,7 +851,7 @@
 {
     switch (value_id) {
     case 'c':       /* current_time() */
-        return rpy_revdb.stop_point_seen;
+        return stopped_time ? stopped_time : rpy_revdb.stop_point_seen;
     case 'f':       /* most_recent_fork() */
         return most_recent_fork;
     case 't':       /* total_time() */

From pypy.commits at gmail.com  Mon Jun 13 09:46:42 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 06:46:42 -0700 (PDT)
Subject: [pypy-commit] pypy hypothesis-apptest: a hypothesis strategy for
 randomizing the results of we_are_jitted()
Message-ID: <575eb942.2457c20a.ec0a.5aab@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: hypothesis-apptest
Changeset: r85126:0485618dc84d
Date: 2016-06-13 15:44 +0200
http://bitbucket.org/pypy/pypy/changeset/0485618dc84d/

Log:	a hypothesis strategy for randomizing the results of we_are_jitted()

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -325,12 +325,42 @@
         hop.exception_cannot_occur()
         return hop.genop('hint', [v, c_hint], resulttype=v.concretetype)
 
-
 def we_are_jitted():
     """ Considered as true during tracing and blackholing,
     so its consquences are reflected into jitted code """
+    global _hypothesis_data
+    if _hypothesis_data is not None:
+        if _hypothesis_data.data.frozen:
+            # outside of test, reset
+            _hypothesis_data = None
+        else:
+            import hypothesis.strategies as strategies
+            return _hypothesis_data.draw(strategies.booleans())
     return False
 
+def _randomize_we_are_jitted_from(data):
+    global _hypothesis_data
+    _hypothesis_data = data
+    return None
+
+def randomized_we_are_jitted_strategy():
+    """ a Hypothesis strategy to test functions that rely on we_are_jitted().
+    At runtime, we_are_jitted() can either return True of False, in a somewhat
+    hard to predict way. The functionality of the interpreter should never
+    really depend on the value of the returned bool. To test this, hypothesis
+    can be used with this strategy, in the following way:
+
+    @given(randomized_we_are_jitted_strategy())
+    def test_something(_ignored):
+        # in here, we_are_jitted() randomly returns True or False.
+        # the test is run many times to try to make it fail.
+
+    (the implementation, however, is a bit of a hack).
+    """
+    from hypothesis import strategies
+    return strategies.builds(_randomize_we_are_jitted_from, strategies.data())
+
+
 _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */',
                                      default=0)
 
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -5,7 +5,7 @@
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
     JitHintError, oopspec, isconstant, conditional_call,
     elidable, unroll_safe, dont_look_inside,
-    enter_portal_frame, leave_portal_frame)
+    enter_portal_frame, leave_portal_frame, randomized_we_are_jitted_strategy)
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.lltypesystem import lltype
@@ -115,6 +115,32 @@
         def f():
             pass
 
+def test_randomized_we_are_jitted():
+    from hypothesis import given
+    def good():
+        if we_are_jitted():
+            return 1
+        return 1
+    counter = [0]
+    def bad():
+        if we_are_jitted():
+            return 2
+        return 1
+    @given(randomized_we_are_jitted_strategy())
+    def test_random_good(_):
+        assert good() == 1
+    test_random_good()
+
+    @given(randomized_we_are_jitted_strategy())
+    def test_random_bad(_):
+        assert bad() == 1
+    try:
+        test_random_bad()
+    except AssertionError:
+        pass
+    else:
+        assert 0, "should have failed!"
+
 class TestJIT(BaseRtypingTest):
     def test_hint(self):
         def f():

From pypy.commits at gmail.com  Mon Jun 13 09:51:43 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 06:51:43 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: (arigo,
 cfbolz): to make the tests check both paths, return True or False
Message-ID: <575eba6f.8aa6c20a.15839.271a@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85127:d3b59f1f5ac9
Date: 2016-05-24 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b59f1f5ac9/

Log:	(arigo, cfbolz): to make the tests check both paths, return True or
	False randomly from we_are_jitted

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1,4 +1,5 @@
 import sys
+import random
 
 import py
 
@@ -381,7 +382,9 @@
 def we_are_jitted():
     """ Considered as true during tracing and blackholing,
     so its consquences are reflected into jitted code """
-    return False
+    # during testing we return something randomly, to emulate the real
+    # behaviour where you can switch to tracing a arbitrary points.
+    return random.random() > 0.5
 
 _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */',
                                      default=0)

From pypy.commits at gmail.com  Mon Jun 13 09:51:45 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 06:51:45 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: deal with TypeErrors in
 we_are_jitted() blocks correctly
Message-ID: <575eba71.cf981c0a.ede76.ffffb318@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85128:4d70d977e2f2
Date: 2016-05-24 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/4d70d977e2f2/

Log:	deal with TypeErrors in we_are_jitted() blocks correctly

diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py
--- a/pypy/objspace/std/callmethod.py
+++ b/pypy/objspace/std/callmethod.py
@@ -48,11 +48,15 @@
     name = None
     if jit.we_are_jitted():
         # compute safeness without reading the type
-        map = w_obj._get_mapdict_map_no_promote()
-        if map is not None and map._type_safe_to_do_getattr():
-            safe = True
-            name = space.str_w(w_name)
-            w_descr = map._type_lookup_safe(name)
+        try:
+            map = w_obj._get_mapdict_map_no_promote()
+        except TypeError:
+            pass
+        else:
+            if map._type_safe_to_do_getattr():
+                safe = True
+                name = space.str_w(w_name)
+                w_descr = map._type_lookup_safe(name)
     else:
         w_type = space.type(w_obj)
         safe = w_type.has_object_getattribute()
@@ -143,10 +147,14 @@
     w_descr = None
     if jit.we_are_jitted():
         # compute safeness without reading the type
-        map = w_obj._get_mapdict_map_no_promote()
-        if map is not None and map._type_safe_to_do_getattr():
-            safe = True
-            w_descr = map._type_lookup_safe(methname)
+        try:
+            map = w_obj._get_mapdict_map_no_promote()
+        except TypeError:
+            pass
+        else:
+            if map._type_safe_to_do_getattr():
+                safe = True
+                w_descr = map._type_lookup_safe(methname)
     else:
         w_type = space.type(w_obj)
         safe = w_type.has_object_getattribute()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -535,11 +535,15 @@
         safe = False
         if jit.we_are_jitted():
             # compute safeness without reading the type
-            map = w_obj._get_mapdict_map_no_promote()
-            if map is not None and map._type_safe_to_do_getattr():
-                safe = True
-                name = self.str_w(w_name)
-                w_descr = map._type_lookup_safe(name)
+            try:
+                map = w_obj._get_mapdict_map_no_promote()
+            except TypeError:
+                pass
+            else:
+                if map._type_safe_to_do_getattr():
+                    safe = True
+                    name = self.str_w(w_name)
+                    w_descr = map._type_lookup_safe(name)
 
         if not safe:
             w_type = self.type(w_obj)

From pypy.commits at gmail.com  Mon Jun 13 09:51:48 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 06:51:48 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: merge default
Message-ID: <575eba74.d12f1c0a.82d90.ffffb5fc@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85129:6ead2971b58b
Date: 2016-06-13 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/6ead2971b58b/

Log:	merge default

diff too long, truncating to 2000 out of 16968 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,7 @@
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -93,9 +93,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,17 +104,20 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas Stührk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -140,7 +143,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +158,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +175,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +187,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,11 +210,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -228,7 +230,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -270,8 +271,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +297,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -328,8 +328,8 @@
 
     def test_13_genexp(self):
         if self.using_gc:
+            gc.enable()
             test_support.gc_collect()
-            gc.enable()
         try:
             self.run_test(generator_example)
             # issue1265: if the trace function contains a generator,
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -6,7 +6,7 @@
 irc_header = "And now for something completely different"
 
 
-def interactive_console(mainmodule=None, quiet=False):
+def interactive_console(mainmodule=None, quiet=False, future_flags=0):
     # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
     # mimics what CPython does in pythonrun.c
     if not hasattr(sys, 'ps1'):
@@ -37,15 +37,17 @@
             raise ImportError
         from pyrepl.simple_interact import run_multiline_interactive_console
     except ImportError:
-        run_simple_interactive_console(mainmodule)
+        run_simple_interactive_console(mainmodule, future_flags=future_flags)
     else:
-        run_multiline_interactive_console(mainmodule)
+        run_multiline_interactive_console(mainmodule, future_flags=future_flags)
 
-def run_simple_interactive_console(mainmodule):
+def run_simple_interactive_console(mainmodule, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat 
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -4,6 +4,9 @@
 subprocess module on Windows.
 """
 
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
 
 # Declare external Win32 functions
 
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.7.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.7.0"
+__version_info__ = (1, 7, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -57,6 +57,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.7.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -332,8 +332,8 @@
     def from_buffer(self, python_buffer):
         """Return a  that points to the data of the
         given Python object, which must support the buffer interface.
-        Note that this is not meant to be used on the built-in types str,
-        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        Note that this is not meant to be used on the built-in types
+        str or unicode (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
         """
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -465,6 +463,7 @@
             else:
                 def __nonzero__(self):
                     return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -35,8 +35,11 @@
                                "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError("Unsupported type: %r.  Please file a bug "
-                                   "if you think it should be." % (commontype,))
+                raise api.FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
             result, quals = parser.parse_type_and_quals(cdecl)   # recursive
 
         assert isinstance(result, model.BaseTypeByIdentity)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -814,7 +814,7 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                    prnt("  (void)((p->%s) | 0);  /* check that '%s.%s' is "
                          "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
@@ -991,7 +991,7 @@
             prnt('static int %s(unsigned long long *o)' % funcname)
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
-            prnt('  *o = (unsigned long long)((%s) << 0);'
+            prnt('  *o = (unsigned long long)((%s) | 0);'
                  '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
@@ -1250,7 +1250,7 @@
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
         s = ('_cffi_prim_int(sizeof(%s), (\n'
-             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '           ((%s)-1) | 0 /* check that %s is an integer type */\n'
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -43,11 +43,13 @@
         return short
     return text
 
-def run_multiline_interactive_console(mainmodule=None):
+def run_multiline_interactive_console(mainmodule=None, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,4 +1,5 @@
-# Empty
+import os
+pypydir = os.path.realpath(os.path.dirname(__file__))
 
 # XXX Should be empty again, soon.
 # XXX hack for win64:
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -1,4 +1,4 @@
-import py, pytest, sys, os, textwrap
+import py, pytest, sys, textwrap
 from inspect import isclass
 
 # pytest settings
@@ -10,8 +10,6 @@
 #
 option = None
 
-pypydir = os.path.realpath(os.path.dirname(__file__))
-
 def braindead_deindent(self):
     """monkeypatch that wont end up doing stupid in the python tokenizer"""
     text = '\n'.join(self.lines)
@@ -78,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -112,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -122,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -155,35 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
-
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
 
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -70,9 +70,6 @@
 bz2
     libbz2
 
-lzma (PyPy3 only)
-    liblzma
-
 pyexpat
     libexpat1
 
@@ -98,11 +95,16 @@
 tk
     tk-dev
 
+lzma (PyPy3 only)
+    liblzma
+
+To run untranslated tests, you need the Boehm garbage collector libgc.
+
 On Debian, this is the command to install all build-time dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev
+    tk-dev libgc-dev liblzma-dev
 
 For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
 
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -13,17 +13,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,9 +63,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -74,31 +74,34 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
+  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Edd Barrett
   Andreas Stührk
-  Edd Barrett
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Paweł Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -110,7 +113,6 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -126,11 +128,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -141,9 +145,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -153,8 +157,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -178,11 +180,11 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -198,7 +200,6 @@
   Lukas Vacek
   Kunal Grover
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
@@ -240,8 +241,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -265,9 +267,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafał Gałczyński
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
    release-5.0.1.rst
@@ -49,6 +50,13 @@
    release-0.6
 
 
+CPython 3.3 compatible versions
+-------------------------------
+
+.. toctree::
+
+   release-pypy3.3-v5.2-alpha1.rst
+
 CPython 3.2 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
    whatsnew-4.0.1.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,15 +53,17 @@
 immediately, but only when (and if) ``myslice`` or ``mylist`` are mutated.
 
 
-Numpy improvements
-------------------
+NumPy rebooted
+--------------
 
-The numpy is rapidly progressing in pypy, so feel free to come to IRC and
-ask for proposed topic. A not necesarilly up-to-date `list of topics`_
-is also available.
+Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
+Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+test suite. We could use help analyzing the failures and fixing them either
+as patches to upstream NumPy, or as fixes to PyPy.
 
-.. _list of topics: https://bitbucket.org/pypy/extradoc/src/extradoc/planning/micronumpy.txt
-
+We also are looking for help in how to hijack NumPy dtype conversion and
+ufunc calls to allow the JIT to make them fast, using our internal _numpypy
+module.
 
 Improving the jitviewer
 ------------------------
diff --git a/pypy/doc/release-pypy2.7-v5.3.0.rst b/pypy/doc/release-pypy2.7-v5.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.0.rst
@@ -0,0 +1,193 @@
+============
+PyPy2.7 v5.3
+============
+
+We have released PyPy2.7 v5.3, about six weeks after PyPy 5.1 and a week after
+`PyPy3.3 v5.2 alpha 1`_, the first PyPy release targetting 3.3
+compatibility. This new PyPy2.7 release includes further improvements for the
+CAPI compatibility layer which we call cpyext. In addtion to complete support
+for lxml, we now pass most (more than 90%) of the upstream numpy test suite,
+and much of SciPy is supported as well.
+
+We updated cffi_ to version 1.7 (small changes, documented here_).
+
+.. _`PyPy3.3 v5.2 alpha 1`: http://morepypy.blogspot.com/2016/05/pypy33-v52-alpha-1-released.html
+.. _cffi: https://cffi.readthedocs.org
+.. _here: http://cffi.readthedocs.io/en/latest/whatsnew.html
+
+You can download the PyPy2.7 v5.3 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.1 released in April 2016)
+=========================================================
+
+* New features:
+
+  * Merge a major expansion of the C-API support in cpyext, here are some of
+    the highlights:
+
+      - allow c-snippet tests to be run with -A so we can verify we are compatible
+      - fix many edge cases exposed by fixing tests to run with -A
+      - issequence() logic matches cpython
+      - make PyStringObject and PyUnicodeObject field names compatible with cpython
+      - add prelminary support for PyDateTime_*
+      - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
+        PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
+        PyAnySet_CheckExact, PyUnicode_Concat, PyDateTime_TZInfo
+      - improve support for PyGILState_Ensure, PyGILState_Release, and thread
+        primitives, also find a case where CPython will allow thread creation
+        before PyEval_InitThreads is run, dissallow on PyPy 
+      - create a PyObject-specific list strategy
+      - rewrite slot assignment for typeobjects
+      - improve tracking of PyObject to rpython object mapping
+      - support tp_as_{number, sequence, mapping, buffer} slots
+      - support ByteArrayObject via the new resizable_list_supporting_raw_ptr
+      - implement PyList_SET_ITEM with CPython's behavior, instead of SetItem's
+      - fix the signature of PyUFunc_FromFuncAndDataAndSignature
+      - implement many PyWhatever_FOO() as a macro taking a `void *`
+
+  * CPyExt tweak: instead of "GIL not held when a CPython C extension module
+    calls PyXxx", we now silently acquire/release the GIL.  Helps with
+    CPython C extension modules that call some PyXxx() functions without
+    holding the GIL (arguably, they are theorically buggy).
+
+  * Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
+    It is a more flexible way to make RPython finalizers. Use this mechanism to
+    clean up handling of ``__del__`` methods, fixing issue #2287
+
+  * Generalize cpyext old-style buffers to more than just str/buffer, add
+    support for mmap
+
+  * Support command line -v to trace import statements
+
+  * Add rposix functions for PyPy3.3 support
+
+  * Give super an __init__ and a simple __new__ for CPython compatibility
+
+  * Revive traceviewer, a tool to use pygame to view traces
+
+* Bug Fixes
+
+  * Fix issue #2277: only special-case two exact lists in zip(), not list
+    subclasses, because an overridden __iter__() should be called (probably)
+
+  * Fix issue #2226: Another tweak in the incremental GC- this should ensure
+    that progress in the major GC occurs quickly enough in all cases.
+
+  * Clarify and refactor documentation on http://doc.pypy.org
+
+  * Use "must be unicode, not %T" in unicodedata TypeErrors.
+
+  * Manually reset sys.settrace() and sys.setprofile() when we're done running.
+    This is not exactly what CPython does, but if we get an exception, unlike
+    CPython, we call functions from the 'traceback' module, and these would
+    call more the trace/profile function.  That's unexpected and can lead
+    to more crashes at this point.
+
+  * Use the appropriate tp_dealloc on a subclass of a builtin type, and call
+    tp_new for a python-sublcass of a C-API type
+
+  * Fix for issue #2285 - rare vmprof segfaults on OS/X
+
+  * Fixed issue #2172 - where a test specified an invalid parameter to mmap on powerpc
+
+  * Fix issue #2311 - grab the `__future__` flags imported in the main script, in
+    `-c`, or in `PYTHON_STARTUP`, and expose them to the `-i` console
+
+  * Issues reported with our previous release were resolved_ after reports from users on
+    our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+    #pypy
+
+* Numpy_:
+
+  * Implement ufunc.outer on numpypy
+
+  * Move PyPy-specific numpy headers to a subdirectory (also changed `the repo`_
+    accordingly)
+
+* Performance improvements:
+
+  * Use bitstrings to compress lists of descriptors that are attached to an
+    EffectInfo
+
+  * Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+    quite some code internally, and allows the JIT to do better
+    optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+    can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+    negative.
+
+  * Copy CPython's 'optimization': ignore __iter__ etc. for `f(**dict_subclass())`
+
+  * Use the __builtin_add_overflow built-ins if they are available
+
+  * Rework the way registers are moved/spilled in before_call()
+
+* Internal refactorings:
+
+  * Refactor code to better support Python3-compatible syntax
+
+  * Document and refactor OperationError -> oefmt
+
+  * Reduce the size of generated C sources during translation by 
+    eliminating many many unused struct declarations (Issue #2281)
+
+  * Remove a number of translation-time options that were not tested and
+    never used. Also fix a performance bug in the method cache
+
+  * Reduce the size of generated code by using the same function objects in
+    all generated subclasses
+
+  * Share cpyext Py* function wrappers according to the signature, shrinking the
+    translated libpypy.so by about 10% (measured without the JIT)
+
+  * Compile c snippets with -Werror, and fix warnings it exposed
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.3.0.html
+.. _Numpy: https://bitbucket.org/pypy/numpy
+.. _`the repo`: https://bitbucket.org/pypy/numpy
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
@@ -0,0 +1,69 @@
+===================
+PyPy3 v5.2 alpha 1
+===================
+
+We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the
+first release of PyPy which targets Python 3.3 (3.3.5) compatibility.
+
+We would like to thank all of the people who donated_ to the `py3k proposal`_
+for supporting the work that went into this and future releases.
+
+You can download the PyPy3.3 v5.2 alpha 1 release here:
+
+    http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2
+
+Highlights
+==========
+
+* Python 3.3.5 support!
+
+  - Being an early alpha release, there are some `missing features`_ such as a
+    `PEP 393-like space efficient string representation`_ and `known issues`_
+    including performance regressions (e.g. issue `#2305`_). The focus for this
+    release has been updating to 3.3 compatibility. Windows is also not yet
+    supported.
+
+* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4).
+
+What is PyPy?
+==============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT
+compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems except Windows
+    (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+Please try it out and let us know what you think. We welcome feedback, we know
+you are using PyPy, please tell us about it!
+
+We'd especially like to thank these people for their contributions to this
+release:
+
+Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey,
+Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others.
+
+Cheers
+
+The PyPy Team
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation
+.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement
+.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29
+.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305
+.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip
+.. _`dynamic languages`: http://pypyjs.org
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -73,6 +73,8 @@
     'Richard Lancaster':['richardlancaster'],
     'William Leslie':['William ML Leslie'],
     'Spenser Bauman':['Spenser Andrew Bauman'],
+    'Raffael Tfirst':['raffael.tfirst at gmail.com'],
+    'timo':['timo at eistee.fritz.box'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,95 +1,33 @@
 =========================
-What's new in PyPy 5.1+
+What's new in PyPy2.7 5.3+
 =========================
 
-.. this is a revision shortly after release-5.1
-.. startrev: aa60332382a1
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
 
-.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
 
-.. branch: gcheader-decl
+.. branch: fix-gen-dfa
 
-Reduce the size of generated C sources.
+Resolves an issue with the generator script to build the dfa for Python syntax.
 
+.. branch: z196-support
 
-.. branch: remove-objspace-options
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
 
-Remove a number of options from the build process that were never tested and
-never set. Fix a performance bug in the method cache.
+.. branch: s390x-5.3-catchup
 
-.. branch: bitstring
+Implement the backend related changes for s390x.
 
-JIT: use bitstrings to compress the lists of read or written descrs
-that we attach to EffectInfo.  Fixes a problem we had in
-remove-objspace-options.
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
 
-.. branch: cpyext-for-merge
+.. branch: testing-cleanup
 
-Update cpyext C-API support After this branch, we are almost able to support 
-upstream numpy via cpyext, so we created (yet another) fork of numpy at 
-github.com/pypy/numpy with the needed changes. Among the significant changes 
-to cpyext:
-  - allow c-snippet tests to be run with -A so we can verify we are compatible
-  - fix many edge cases exposed by fixing tests to run with -A
-  - issequence() logic matches cpython
-  - make PyStringObject and PyUnicodeObject field names compatible with cpython
-  - add prelminary support for PyDateTime_*
-  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
-    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
-  - PyAnySet_CheckExact, PyUnicode_Concat
-  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
-    primitives, also find a case where CPython will allow thread creation
-    before PyEval_InitThreads is run, dissallow on PyPy 
-  - create a PyObject-specific list strategy
-  - rewrite slot assignment for typeobjects
-  - improve tracking of PyObject to rpython object mapping
-  - support tp_as_{number, sequence, mapping, buffer} slots
+Simplify handling of interp-level tests and make it more forward-
+compatible.
 
-(makes the pypy-c bigger; this was fixed subsequently by the
-share-cpyext-cpython-api branch)
-
-.. branch: share-mapdict-methods-2
-
-Reduce generated code for subclasses by using the same function objects in all
-generated subclasses.
-
-.. branch: share-cpyext-cpython-api
-
-.. branch: cpyext-auto-gil
-
-CPyExt tweak: instead of "GIL not held when a CPython C extension module
-calls PyXxx", we now silently acquire/release the GIL.  Helps with
-CPython C extension modules that call some PyXxx() functions without
-holding the GIL (arguably, they are theorically buggy).
-
-.. branch: cpyext-test-A
-
-Get the cpyext tests to pass with "-A" (i.e. when tested directly with
-CPython).
-
-.. branch: oefmt
-
-.. branch: cpyext-werror
-
-Compile c snippets with -Werror in cpyext
-
-.. branch: gc-del-3
-
-Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
-It is a more flexible way to make RPython finalizers.
-
-.. branch: unpacking-cpython-shortcut
-
-.. branch: cleanups
-
-.. branch: cpyext-more-slots
-
-.. branch: use-gc-del-3
-
-Use the new rgc.FinalizerQueue mechanism to clean up the handling of
-``__del__`` methods.  Fixes notably issue #2287.  (All RPython
-subclasses of W_Root need to use FinalizerQueue now.)
-
-.. branch: ufunc-outer
-
-Implement ufunc.outer on numpypy
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -0,0 +1,145 @@
+=========================
+What's new in PyPy2.7 5.3
+=========================
+
+.. this is a revision shortly after release-5.1
+.. startrev: aa60332382a1
+
+.. branch: techtonik/introductionrst-simplify-explanation-abo-1460879168046
+
+.. branch: gcheader-decl
+
+Reduce the size of generated C sources.
+
+
+.. branch: remove-objspace-options
+
+Remove a number of options from the build process that were never tested and
+never set. Fix a performance bug in the method cache.
+
+.. branch: bitstring
+
+JIT: use bitstrings to compress the lists of read or written descrs
+that we attach to EffectInfo.  Fixes a problem we had in
+remove-objspace-options.
+
+.. branch: cpyext-for-merge
+
+Update cpyext C-API support After this branch, we are almost able to support 
+upstream numpy via cpyext, so we created (yet another) fork of numpy at 
+github.com/pypy/numpy with the needed changes. Among the significant changes 
+to cpyext:
+  - allow c-snippet tests to be run with -A so we can verify we are compatible
+  - fix many edge cases exposed by fixing tests to run with -A
+  - issequence() logic matches cpython
+  - make PyStringObject and PyUnicodeObject field names compatible with cpython
+  - add prelminary support for PyDateTime_*
+  - support PyComplexObject, PyFloatObject, PyDict_Merge, PyDictProxy,
+    PyMemoryView_*, _Py_HashDouble, PyFile_AsFile, PyFile_FromFile,
+  - PyAnySet_CheckExact, PyUnicode_Concat
+  - improve support for PyGILState_Ensure, PyGILState_Release, and thread
+    primitives, also find a case where CPython will allow thread creation
+    before PyEval_InitThreads is run, dissallow on PyPy 
+  - create a PyObject-specific list strategy
+  - rewrite slot assignment for typeobjects
+  - improve tracking of PyObject to rpython object mapping
+  - support tp_as_{number, sequence, mapping, buffer} slots
+
+(makes the pypy-c bigger; this was fixed subsequently by the
+share-cpyext-cpython-api branch)
+
+.. branch: share-mapdict-methods-2
+
+Reduce generated code for subclasses by using the same function objects in all
+generated subclasses.
+
+.. branch: share-cpyext-cpython-api
+
+.. branch: cpyext-auto-gil
+
+CPyExt tweak: instead of "GIL not held when a CPython C extension module
+calls PyXxx", we now silently acquire/release the GIL.  Helps with
+CPython C extension modules that call some PyXxx() functions without
+holding the GIL (arguably, they are theorically buggy).
+
+.. branch: cpyext-test-A
+
+Get the cpyext tests to pass with "-A" (i.e. when tested directly with
+CPython).
+
+.. branch: oefmt
+
+.. branch: cpyext-werror
+
+Compile c snippets with -Werror in cpyext
+
+.. branch: gc-del-3
+
+Add rgc.FinalizerQueue, documented in pypy/doc/discussion/finalizer-order.rst.
+It is a more flexible way to make RPython finalizers.
+
+.. branch: unpacking-cpython-shortcut
+
+.. branch: cleanups
+
+.. branch: cpyext-more-slots
+
+.. branch: use-gc-del-3
+
+Use the new rgc.FinalizerQueue mechanism to clean up the handling of
+``__del__`` methods.  Fixes notably issue #2287.  (All RPython
+subclasses of W_Root need to use FinalizerQueue now.)
+
+.. branch: ufunc-outer
+
+Implement ufunc.outer on numpypy
+
+.. branch: verbose-imports
+
+Support ``pypy -v``: verbose imports.  It does not log as much as
+cpython, but it should be enough to help when debugging package layout
+problems.
+
+.. branch: cpyext-macros-cast
+
+Fix some warnings when compiling CPython C extension modules
+
+.. branch: syntax_fix
+
+.. branch: remove-raisingops
+
+Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+quite some code internally, and allows the JIT to do better
+optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+negative.
+
+.. branch: cpyext-old-buffers
+
+Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
+
+.. branch: numpy-includes
+
+Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
+This allows building upstream numpy and scipy in pypy via cpyext
+
+.. branch: traceviewer-common-merge-point-formats
+
+Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
+
+.. branch: cpyext-pickle
+
+Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
+at cpyext import time
+
+.. branch: nonmovable-list
+
+Add a way to ask "give me a raw pointer to this list's
+items".  Only for resizable lists of primitives.  Turns the GcArray
+nonmovable, possibly making a copy of it first.
+
+.. branch: cpyext-ext
+
+Finish the work already partially merged in cpyext-for-merge. Adds support
+for ByteArrayObject using the nonmovable-list, which also enables
+buffer(bytearray()) 
diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
@@ -0,0 +1,10 @@
+=================================
+What's new in PyPy3 5.1.1 alpha 1
+=================================
+
+.. A recent revision, ignoring all other branches for this release
+.. startrev: 29d14733e007
+
+.. branch: py3.3
+
+Python 3.3 compatibility
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -238,6 +238,15 @@
 for use. The release packaging script will pick up the tcltk runtime in the lib
 directory and put it in the archive.
 
+The lzma compression library
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.3 ship with CFFI wrappers for the lzma library, which can be
+downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
+5.0.5, a prebuilt version can be downloaded from
+http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
+http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
+
 
 Using the mingw compiler
 ------------------------
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -9,7 +9,7 @@
 from rpython.config.config import to_optparse, make_dict, SUPPRESS_USAGE
 from rpython.config.config import ConflictConfigError
 from pypy.tool.option import make_objspace
-from pypy.conftest import pypydir
+from pypy import pypydir
 from rpython.rlib import rthread
 from pypy.module.thread import os_thread
 
@@ -293,7 +293,7 @@
             self.hack_for_cffi_modules(driver)
 
         return self.get_entry_point(config)
-    
+
     def hack_for_cffi_modules(self, driver):
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
@@ -320,7 +320,7 @@
             while not basedir.join('include').exists():
                 _basedir = basedir.dirpath()
                 if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo', 
+                    raise ValueError('interpreter %s not inside pypy repo',
                                      str(exename))
                 basedir = _basedir
             modules = self.config.objspace.usemodules.getpaths()
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,7 +2,7 @@
 # This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
-# Missing vs CPython: -d, -t, -v, -x, -3
+# Missing vs CPython: -d, -t, -x, -3
 USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
 -B     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
@@ -19,6 +19,8 @@
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
 -S     : don't imply 'import site' on initialization
 -u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x
+-v     : verbose (trace import statements); also PYTHONVERBOSE=x
+         can be supplied multiple times to increase verbosity
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
@@ -529,6 +531,7 @@
                      warnoptions,
                      unbuffered,
                      ignore_environment,
+                     verbose,
                      **ignored):
     # with PyPy in top of CPython we can only have around 100
     # but we need more in the translated PyPy for the compiler package
@@ -580,6 +583,12 @@
         if hasattr(signal, 'SIGXFSZ'):
             signal.signal(signal.SIGXFSZ, signal.SIG_IGN)
 
+    # Pre-load the default encoder (controlled by PYTHONIOENCODING) now.
+    # This is needed before someone mucks up with sys.path (or even adds
+    # a unicode string to it, leading to infinite recursion when we try
+    # to encode it during importing).  Note: very obscure.  Issue #2314.
+    str(u'')
+
     def inspect_requested():
         # We get an interactive prompt in one of the following three cases:
         #
@@ -600,6 +609,11 @@
                 ((inspect or (readenv and real_getenv('PYTHONINSPECT')))
                  and sys.stdin.isatty()))
 
+    try:
+        from _ast import PyCF_ACCEPT_NULL_BYTES
+    except ImportError:
+        PyCF_ACCEPT_NULL_BYTES = 0
+    future_flags = [0]
     success = True
 
     try:
@@ -610,7 +624,9 @@
 
             @hidden_applevel
             def run_it():
-                exec run_command in mainmodule.__dict__
+                co_cmd = compile(run_command, '', 'exec')
+                exec co_cmd in mainmodule.__dict__
+                future_flags[0] = co_cmd.co_flags
             success = run_toplevel(run_it)
         elif run_module:
             # handle the "-m" command
@@ -622,11 +638,6 @@
             # handle the case where no command/filename/module is specified
             # on the command-line.
 
-            try:
-                from _ast import PyCF_ACCEPT_NULL_BYTES
-            except ImportError:
-                PyCF_ACCEPT_NULL_BYTES = 0
-
             # update sys.path *after* loading site.py, in case there is a
             # "site.py" file in the script's directory. Only run this if we're
             # executing the interactive prompt, if we're running a script we
@@ -653,6 +664,7 @@
                                                         'exec',
                                                         PyCF_ACCEPT_NULL_BYTES)
                             exec co_python_startup in mainmodule.__dict__
+                            future_flags[0] = co_python_startup.co_flags
                         mainmodule.__file__ = python_startup
                         run_toplevel(run_it)
                         try:
@@ -663,11 +675,14 @@
                 inspect = True
             else:
                 # If not interactive, just read and execute stdin normally.
+                if verbose:
+                    print_banner(not no_site)
                 @hidden_applevel
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '', 'exec',
                                        PyCF_ACCEPT_NULL_BYTES)
                     exec co_stdin in mainmodule.__dict__
+                    future_flags[0] = co_stdin.co_flags
                 mainmodule.__file__ = ''
                 success = run_toplevel(run_it)
         else:
@@ -697,7 +712,20 @@
                     args = (runpy._run_module_as_main, '__main__', False)
                 else:
                     # no.  That's the normal path, "pypy stuff.py".
-                    args = (execfile, filename, mainmodule.__dict__)
+                    # This includes the logic from execfile(), tweaked
+                    # to grab the future_flags at the end.
+                    @hidden_applevel
+                    def run_it():
+                        f = file(filename, 'rU')
+                        try:
+                            source = f.read()
+                        finally:
+                            f.close()
+                        co_main = compile(source.rstrip()+"\n", filename,
+                                          'exec', PyCF_ACCEPT_NULL_BYTES)
+                        exec co_main in mainmodule.__dict__
+                        future_flags[0] = co_main.co_flags
+                    args = (run_it,)
             success = run_toplevel(*args)
 
     except SystemExit as e:
@@ -710,12 +738,21 @@
     # start a prompt if requested
     if inspect_requested():
         try:
+            import __future__
             from _pypy_interact import interactive_console
             pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info)
             irc_topic = pypy_version_info[3] != 'final' or (
                             readenv and os.getenv('PYPY_IRC_TOPIC'))
+            flags = 0
+            for fname in __future__.all_feature_names:
+                feature = getattr(__future__, fname)
+                if future_flags[0] & feature.compiler_flag:
+                    flags |= feature.compiler_flag
+            kwds = {}
+            if flags:
+                kwds['future_flags'] = flags
             success = run_toplevel(interactive_console, mainmodule,
-                                   quiet=not irc_topic)
+                                   quiet=not irc_topic, **kwds)
         except SystemExit as e:
             status = e.code
         else:
@@ -724,10 +761,10 @@
     return status
 
 def print_banner(copyright):
-    print 'Python %s on %s' % (sys.version, sys.platform)
+    print >> sys.stderr, 'Python %s on %s' % (sys.version, sys.platform)
     if copyright:
-        print ('Type "help", "copyright", "credits" or '
-               '"license" for more information.')
+        print >> sys.stderr, ('Type "help", "copyright", "credits" or '
+                              '"license" for more information.')
 
 STDLIB_WARNING = """\
 debug: WARNING: Library path not found, using compiled-in sys.path.
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -564,7 +564,6 @@
             self.emit_jump(ops.JUMP_FORWARD, end)
             self.use_next_block(next_except)
         self.emit_op(ops.END_FINALLY)   # this END_FINALLY will always re-raise
-        self.is_dead_code()
         self.use_next_block(otherwise)
         self.visit_sequence(te.orelse)
         self.use_next_block(end)
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -1,8 +1,8 @@
 from pypy.interpreter.astcompiler import ast
 class TestAstToObject:
     def test_types(self, space):
-        assert space.is_true(space.issubtype(
-                ast.get(space).w_Module, ast.get(space).w_mod))
+        assert space.issubtype_w(
+                ast.get(space).w_Module, ast.get(space).w_mod)
                                   
     def test_num(self, space):
         value = space.wrap(42)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -458,14 +458,17 @@
         decl = str(decl) + "\n"
         yield self.st, decl, 'x', (1, 2, 3, 4)
 
+    def test_closure_error(self):
         source = """if 1:
         def f(a):
             del a
             def x():
                 a
         """
-        exc = py.test.raises(SyntaxError, self.run, source).value
-        assert exc.msg == "Can't delete variable used in nested scopes: 'a'"
+        with py.test.raises(SyntaxError) as excinfo:
+            self.run(source)
+        msg = excinfo.value.msg
+        assert msg == "Can't delete variable used in nested scopes: 'a'"
 
     def test_try_except_finally(self):
         yield self.simple_test, """
@@ -879,7 +882,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -903,19 +919,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
@@ -1111,7 +1115,7 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
-        
+
     def test_list_comprehension(self):
         source = "def f(): [i for i in l]"
         source2 = "def f(): [i for i in l for j in l]"
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -245,6 +245,9 @@
     def unicode_w(self, space):
         self._typed_unwrap_error(space, "unicode")
 
+    def bytearray_list_of_chars_w(self, space):
+        self._typed_unwrap_error(space, "bytearray")
+
     def int_w(self, space, allow_conversion=True):
         # note that W_IntObject.int_w has a fast path and W_FloatObject.int_w
         # raises w_TypeError
@@ -1217,7 +1220,7 @@
 
     def abstract_issubclass_w(self, w_cls1, w_cls2):
         # Equivalent to 'issubclass(cls1, cls2)'.
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def abstract_isinstance_w(self, w_obj, w_cls):
         # Equivalent to 'isinstance(obj, cls)'.
@@ -1239,16 +1242,16 @@
     def exception_is_valid_obj_as_class_w(self, w_obj):
         if not self.isinstance_w(w_obj, self.w_type):
             return False
-        return self.is_true(self.issubtype(w_obj, self.w_BaseException))
+        return self.issubtype_w(w_obj, self.w_BaseException)
 
     def exception_is_valid_class_w(self, w_cls):
-        return self.is_true(self.issubtype(w_cls, self.w_BaseException))
+        return self.issubtype_w(w_cls, self.w_BaseException)
 
     def exception_getclass(self, w_obj):
         return self.type(w_obj)
 
     def exception_issubclass_w(self, w_cls1, w_cls2):
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def new_exception_class(self, *args, **kwargs):
         "NOT_RPYTHON; convenience method to create excceptions in modules"
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -205,7 +205,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    map = {"'" : (singleDFA, states_singleDFA),
+           '"' : (doubleDFA, states_doubleDFA),
            "r" : None,
            "R" : None,
            "u" : None,
@@ -257,25 +265,30 @@
     for uniPrefix in ("", "u", "U", "b", "B", ):
         for rawPrefix in ("", "r", "R"):
             prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
+            map[prefix + "'''"] = (single3DFA, states_single3DFA)
+            map[prefix + '"""'] = (double3DFA, states_double3DFA)
     return map
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append('\n')
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -298,22 +311,28 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap["\""]
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.genpytokenize import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -6,7 +6,7 @@
 import sys, os, re, runpy, subprocess
 from rpython.tool.udir import udir
 from contextlib import contextmanager
-from pypy.conftest import pypydir
+from pypy import pypydir
 from lib_pypy._pypy_interact import irc_header
 
 try:
@@ -76,6 +76,11 @@
     print 'Goodbye2'   # should not be reached
     """)
 
+script_with_future = getscript("""
+    from __future__ import division
+    from __future__ import print_function
+    """)
+
 
 class TestParseCommandLine:
     def check_options(self, options, sys_argv, **expected):
@@ -286,7 +291,7 @@
         child.expect('>>>')   # banner
         if irc_topic:
             assert irc_header in child.before
-        else:    
+        else:
             assert irc_header not in child.before
 
     def test_help(self):
@@ -445,6 +450,31 @@
         finally:
             os.environ['PYTHONSTARTUP'] = old
 
+    def test_future_in_executed_script(self):
+        child = self.spawn(['-i', script_with_future])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_python_startup(self, monkeypatch):
+        monkeypatch.setenv('PYTHONSTARTUP', script_with_future)
+        child = self.spawn([])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_cmd(self):
+        child = self.spawn(['-i', '-c', 'from __future__ import division'])
+        child.expect('>>> ')
+        child.sendline('x=1; x/2; 3/4')
+        child.expect('0.5')
+        child.expect('0.75')
+
+    def test_cmd_co_name(self):

From pypy.commits at gmail.com  Mon Jun 13 09:51:50 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 06:51:50 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: merge
Message-ID: <575eba76.8f1d1c0a.7bc6a.74df@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85130:febecdcd73b2
Date: 2016-06-13 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/febecdcd73b2/

Log:	merge

diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py
--- a/pypy/objspace/std/callmethod.py
+++ b/pypy/objspace/std/callmethod.py
@@ -48,11 +48,15 @@
     name = None
     if jit.we_are_jitted():
         # compute safeness without reading the type
-        map = w_obj._get_mapdict_map_no_promote()
-        if map is not None and map._type_safe_to_do_getattr():
-            safe = True
-            name = space.str_w(w_name)
-            w_descr = map._type_lookup_safe(name)
+        try:
+            map = w_obj._get_mapdict_map_no_promote()
+        except TypeError:
+            pass
+        else:
+            if map._type_safe_to_do_getattr():
+                safe = True
+                name = space.str_w(w_name)
+                w_descr = map._type_lookup_safe(name)
     else:
         w_type = space.type(w_obj)
         safe = w_type.has_object_getattribute()
@@ -143,10 +147,14 @@
     w_descr = None
     if jit.we_are_jitted():
         # compute safeness without reading the type
-        map = w_obj._get_mapdict_map_no_promote()
-        if map is not None and map._type_safe_to_do_getattr():
-            safe = True
-            w_descr = map._type_lookup_safe(methname)
+        try:
+            map = w_obj._get_mapdict_map_no_promote()
+        except TypeError:
+            pass
+        else:
+            if map._type_safe_to_do_getattr():
+                safe = True
+                w_descr = map._type_lookup_safe(methname)
     else:
         w_type = space.type(w_obj)
         safe = w_type.has_object_getattribute()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -535,11 +535,15 @@
         safe = False
         if jit.we_are_jitted():
             # compute safeness without reading the type
-            map = w_obj._get_mapdict_map_no_promote()
-            if map is not None and map._type_safe_to_do_getattr():
-                safe = True
-                name = self.str_w(w_name)
-                w_descr = map._type_lookup_safe(name)
+            try:
+                map = w_obj._get_mapdict_map_no_promote()
+            except TypeError:
+                pass
+            else:
+                if map._type_safe_to_do_getattr():
+                    safe = True
+                    name = self.str_w(w_name)
+                    w_descr = map._type_lookup_safe(name)
 
         if not safe:
             w_type = self.type(w_obj)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1,4 +1,5 @@
 import sys
+import random
 
 import py
 
@@ -381,7 +382,9 @@
 def we_are_jitted():
     """ Considered as true during tracing and blackholing,
     so its consquences are reflected into jitted code """
-    return False
+    # during testing we return something randomly, to emulate the real
+    # behaviour where you can switch to tracing a arbitrary points.
+    return random.random() > 0.5
 
 _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */',
                                      default=0)

From pypy.commits at gmail.com  Mon Jun 13 10:45:58 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 07:45:58 -0700 (PDT)
Subject: [pypy-commit] pypy hypothesis-apptest: unbreak regular execution
Message-ID: <575ec726.430ac20a.33013.7920@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: hypothesis-apptest
Changeset: r85131:8261a2709a7c
Date: 2016-06-13 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/8261a2709a7c/

Log:	unbreak regular execution

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -325,6 +325,8 @@
         hop.exception_cannot_occur()
         return hop.genop('hint', [v, c_hint], resulttype=v.concretetype)
 
+_hypothesis_data = None
+
 def we_are_jitted():
     """ Considered as true during tracing and blackholing,
     so its consquences are reflected into jitted code """
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -121,11 +121,12 @@
         if we_are_jitted():
             return 1
         return 1
-    counter = [0]
     def bad():
         if we_are_jitted():
             return 2
         return 1
+    assert good() == 1
+    assert bad() == 1
     @given(randomized_we_are_jitted_strategy())
     def test_random_good(_):
         assert good() == 1

From pypy.commits at gmail.com  Mon Jun 13 11:26:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 08:26:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2324: fix for
 bytearray().replace('a', 'ab')
Message-ID: <575ed0ba.07ecc20a.13dc0.ffff891c@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85132:41928beccff9
Date: 2016-06-13 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/41928beccff9/

Log:	Issue #2324: fix for bytearray().replace('a', 'ab')

	We have too much mess and code duplication around here.

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -162,7 +162,8 @@
             buffer = _get_buffer(space, w_sub)
             res = count(value, buffer, start, end)
 
-        return space.wrap(max(res, 0))
+        assert res >= 0
+        return space.wrap(res)
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -211,6 +211,7 @@
 
         check(bytearray('abc').replace('b', bytearray('d')), 'adc')
         check(bytearray('abc').replace('b', 'd'), 'adc')
+        check(bytearray('').replace('a', 'ab'), '')
 
         check(bytearray('abc').upper(), 'ABC')
         check(bytearray('ABC').lower(), 'abc')
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -291,6 +291,7 @@
     return _search(value, other, start, end, SEARCH_COUNT)
 
 # -------------- substring searching helper ----------------
+# XXX a lot of code duplication with lltypesystem.rstr :-(
 
 SEARCH_COUNT = 0
 SEARCH_FIND = 1
@@ -309,6 +310,8 @@
     if end > len(value):
         end = len(value)
     if start > end:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     count = 0
@@ -326,6 +329,8 @@
     w = n - m
 
     if w < 0:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     mlast = m - 1
@@ -570,18 +575,20 @@
 
 class ByteListBuilder(object):
     def __init__(self, init_size=INIT_SIZE):
+        assert init_size >= 0
         self.l = newlist_hint(init_size)
 
     @specialize.argtype(1)
     def append(self, s):
+        l = self.l
         for c in s:
-            self.l.append(c)
+            l.append(c)
 
     @specialize.argtype(1)
     def append_slice(self, s, start, end):
-        assert 0 <= start <= end <= len(s)
-        for c in s[start:end]:
-            self.l.append(c)
+        l = self.l
+        for i in xrange(start, end):
+            l.append(s[i])
 
     def append_multiple_char(self, c, times):
         assert isinstance(c, str)
@@ -589,8 +596,9 @@
 
     def append_charpsize(self, s, size):
         assert size >= 0
+        l = self.l
         for i in xrange(size):
-            self.l.append(s[i])
+            l.append(s[i])
 
     def build(self):
         return self.l
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -231,6 +231,10 @@
     check_search(count, 'one two three', 'e', 0, 1, res=0)
     check_search(count, 'one two three', '', 0, 13, res=14)
 
+    check_search(count, '', 'ab', 0, 0, res=0)
+    check_search(count, 'a', 'ab', 0, 1, res=0)
+    check_search(count, 'ac', 'ab', 0, 2, res=0)
+
 
 class TestTranslates(BaseRtypingTest):
     def test_split_rsplit(self):
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -972,6 +972,13 @@
             s.count(s, -10)
         py.test.raises(AnnotatorError, self.interpret, f, ())
 
+    def test_count_in_empty_string(self):
+        const = self.const
+        def fn():
+            return const('').count(const('ab'))
+        res = self.interpret(fn, [])
+        assert res == 0
+
     def test_getitem_exc(self):
         const = self.const
         def f(x):

From pypy.commits at gmail.com  Mon Jun 13 11:31:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Jun 2016 08:31:51 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: Don't attempt to modify
 sys.filesystemencoding; skip tests instead if it's not good enough
Message-ID: <575ed1e7.6a56c20a.b129a.ffff9540@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85133:0814a8d50e58
Date: 2016-06-13 16:30 +0100
http://bitbucket.org/pypy/pypy/changeset/0814a8d50e58/

Log:	Don't attempt to modify sys.filesystemencoding; skip tests instead
	if it's not good enough

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1180,25 +1180,19 @@
             res = os.system(cmd)
             assert res == 0
 
+ at py.test.fixture
+def check_fsencoding(space, pytestconfig):
+    if pytestconfig.getvalue('runappdirect'):
+        fsencoding = sys.getfilesystemencoding()
+    else:
+        fsencoding = space.sys.filesystemencoding
+    try:
+        u"ą".encode(fsencoding)
+    except UnicodeEncodeError:
+        py.test.skip("encoding not good enough")
 
+ at py.test.mark.usefixtures('check_fsencoding')
 class AppTestPosixUnicode:
-    def setup_class(cls):
-        if cls.runappdirect:
-            # Can't change encoding
-            try:
-                u"ą".encode(sys.getfilesystemencoding())
-            except UnicodeEncodeError:
-                py.test.skip("encoding not good enough")
-        else:
-            cls.save_fs_encoding = cls.space.sys.filesystemencoding
-            cls.space.sys.filesystemencoding = "utf-8"
-
-    def teardown_class(cls):
-        try:
-            cls.space.sys.filesystemencoding = cls.save_fs_encoding
-        except AttributeError:
-            pass
-
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
         import posix

From pypy.commits at gmail.com  Mon Jun 13 11:51:37 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Jun 2016 08:51:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge branch 'testing-cleanup-py3k'
Message-ID: <575ed689.820b1c0a.5988e.fffff1d5@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85134:8028f6e5a9c9
Date: 2016-06-13 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/8028f6e5a9c9/

Log:	Merge branch 'testing-cleanup-py3k'

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -94,6 +94,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -128,9 +142,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -138,11 +149,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -171,28 +178,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -965,7 +965,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -989,19 +1002,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -694,13 +694,11 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket', 'array']}
 
     def setup_method(self, method):
-        w_HOST = space.wrap(self.HOST)
-        self.w_serv = space.appexec([w_HOST],
+        w_HOST = self.space.wrap(self.HOST)
+        self.w_serv = self.space.appexec([w_HOST],
             '''(HOST):
             import _socket
             serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
@@ -711,7 +709,7 @@
 
     def teardown_method(self, method):
         if hasattr(self, 'w_serv'):
-            space.appexec([self.w_serv], '(serv): serv.close()')
+            self.space.appexec([self.w_serv], '(serv): serv.close()')
             self.w_serv = None
 
     def test_timeout(self):
@@ -830,8 +828,7 @@
 
 
 class AppTestErrno:
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -3,8 +3,9 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
+    spaceconfig = {'usemodules': ['_vmprof', 'struct']}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1')))
         cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2')))
 
@@ -17,7 +18,7 @@
         import struct, sys, gc
 
         WORD = struct.calcsize('l')
-        
+
         def count(s):
             i = 0
             count = 0
@@ -44,7 +45,7 @@
                 else:
                     raise AssertionError(s[i])
             return count
-        
+
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -10,15 +10,15 @@
 from rpython.translator.c.test.test_extfunc import need_sparse_files
 from rpython.rlib import rposix
 
+USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time']
+# py3k os.open uses subprocess, requiring the following per platform
+if os.name != 'nt':
+    USEMODULES += ['fcntl', 'select', '_posixsubprocess']
+else:
+    USEMODULES += ['_rawffi', 'thread']
 
 def setup_module(mod):
-    usemodules = ['binascii', 'posix', 'signal', 'struct', 'time']
-    # py3k os.open uses subprocess, requiring the following per platform
-    if os.name != 'nt':
-        usemodules += ['fcntl', 'select', '_posixsubprocess']
-    else:
-        usemodules += ['_rawffi', 'thread']
-    mod.space = gettestobjspace(usemodules=usemodules)
+    mod.space = gettestobjspace(usemodules=USEMODULES)
     mod.path = udir.join('posixtestfile.txt')
     mod.path.write("this is a test")
     mod.path2 = udir.join('test_posix2-')
@@ -48,9 +48,10 @@
 
 
 class AppTestPosix:
+    spaceconfig = {'usemodules': USEMODULES}
 
     def setup_class(cls):
-        cls.space = space
+        space = cls.space
         cls.w_runappdirect = space.wrap(cls.runappdirect)
         cls.w_posix = space.appexec([], GET_POSIX)
         cls.w_os = space.appexec([], "(): import os as m ; return m")
@@ -1128,14 +1129,11 @@
 
 class AppTestEnvironment(object):
     def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name)
-        cls.w_os = space.appexec([], "(): import os; return os")
         cls.w_path = space.wrap(str(path))
 
     def test_environ(self):
-        import sys
-        environ = self.posix.environ
+        import sys, posix
+        environ = posix.environ
         item_type = str if sys.platform.startswith('win') else bytes
         for k, v in environ.items():
             assert type(k) is item_type
@@ -1147,7 +1145,7 @@
 
     @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive')
     def test_environ_nonascii(self):
-        import sys
+        import sys, os
         name, value = 'PYPY_TEST_日本', 'foobar日本'
         if not sys.platform == 'win32':
             fsencoding = sys.getfilesystemencoding()
@@ -1158,7 +1156,6 @@
                     skip("Requires %s.encode(sys.getfilesystemencoding(), "
                          "'surogateescape') to succeed (or win32)" % ascii(s))
 
-        os = self.os
         os.environ[name] = value
         assert os.environ[name] == value
         assert os.getenv(name) == value
@@ -1168,7 +1165,7 @@
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
-            os = self.os
+            import os
             os.unsetenv("XYZABC") #does not raise
             try:
                 os.environ["ABCABC"]
@@ -1183,45 +1180,40 @@
             res = os.system(cmd)
             assert res == 0
 
+ at py.test.fixture
+def check_fsencoding(space, pytestconfig):
+    if pytestconfig.getvalue('runappdirect'):
+        fsencoding = sys.getfilesystemencoding()
+    else:
+        fsencoding = space.sys.filesystemencoding
+    try:
+        u"ą".encode(fsencoding)
+    except UnicodeEncodeError:
+        py.test.skip("encoding not good enough")
 
+ at py.test.mark.usefixtures('check_fsencoding')
 class AppTestPosixUnicode:
-    def setup_class(cls):
-        cls.space = space
-        cls.w_posix = space.appexec([], GET_POSIX)
-        if cls.runappdirect:
-            # Can't change encoding
-            try:
-                u"ą".encode(sys.getfilesystemencoding())
-            except UnicodeEncodeError:
-                py.test.skip("encoding not good enough")
-        else:
-            cls.save_fs_encoding = space.sys.filesystemencoding
-            space.sys.filesystemencoding = "utf-8"
-
-    def teardown_class(cls):
-        try:
-            cls.space.sys.filesystemencoding = cls.save_fs_encoding
-        except AttributeError:
-            pass
-
     def test_stat_unicode(self):
         # test that passing unicode would not raise UnicodeDecodeError
+        import posix
         try:
-            self.posix.stat("ą")
+            posix.stat(u"ą")
         except OSError:
             pass
 
     def test_open_unicode(self):
         # Ensure passing unicode doesn't raise UnicodeEncodeError
+        import posix
         try:
-            self.posix.open("ą", self.posix.O_WRONLY)
+            posix.open(u"ą", posix.O_WRONLY)
         except OSError:
             pass
 
     def test_remove_unicode(self):
         # See 2 above ;)
+        import posix
         try:
-            self.posix.remove("ą")
+            posix.remove(u"ą")
         except OSError:
             pass
 
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -17,7 +17,6 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 from rpython.tool.udir import udir
 from pypy import pypydir
-from pypy.conftest import PyPyClassCollector
 from inspect import getmro
 
 pypyroot = os.path.dirname(pypydir)
@@ -33,7 +32,6 @@
     def __init__(self, excinfo):
         self.excinfo = excinfo
 
-marker = py.test.mark.applevel
 
 def py3k_repr(value):
     "return the repr() that py3k would give for an object."""
@@ -199,10 +197,6 @@
 
 
 class AppTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(AppTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
     def _prunetraceback(self, traceback):
         return traceback
 
@@ -303,7 +297,7 @@
             self.w_instance = space.call_function(w_class)
 
 
-class AppClassCollector(PyPyClassCollector):
+class AppClassCollector(py.test.Class):
     Instance = AppClassInstance
 
     def setup(self):
diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py
deleted file mode 100644
--- a/pypy/tool/pytest/inttest.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Collects and executes interpreter-level tests.
-#
-# Most pypy tests are of this kind.
-
-import py
-import sys
-from pypy.interpreter.error import OperationError
-from pypy.conftest import PyPyClassCollector
-
-
-def check_keyboard_interrupt(e):
-    # we cannot easily convert w_KeyboardInterrupt to KeyboardInterrupt
-    # in general without a space -- here is an approximation
-    try:
-        if e.w_type.name == 'KeyboardInterrupt':
-            tb = sys.exc_info()[2]
-            raise KeyboardInterrupt, KeyboardInterrupt(), tb
-    except AttributeError:
-        pass
-
-
-marker = py.test.mark.interplevel
-
-
-class IntTestFunction(py.test.collect.Function):
-    def __init__(self, *args, **kwargs):
-        super(IntTestFunction, self).__init__(*args, **kwargs)
-        self._request.applymarker(marker)
-
-    def runtest(self):
-        try:
-            super(IntTestFunction, self).runtest()
-        except OperationError as e:
-            check_keyboard_interrupt(e)
-            raise
-        except Exception as e:
-            cls = e.__class__
-            while cls is not Exception:
-                if cls.__name__ == 'DistutilsPlatformError':
-                    from distutils.errors import DistutilsPlatformError
-                    if isinstance(e, DistutilsPlatformError):
-                        py.test.skip('%s: %s' % (e.__class__.__name__, e))
-                cls = cls.__bases__[0]
-            raise
-
-
-class IntInstanceCollector(py.test.collect.Instance):
-    Function = IntTestFunction
-
-
-class IntClassCollector(PyPyClassCollector):
-    Instance = IntInstanceCollector
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -27,11 +27,11 @@
     result = testdir.runpytest("--collectonly")
     assert result.ret == 0
     result.stdout.fnmatch_lines([
-        "*IntTestFunction*test_func*",
-        "*IntClassCollector*TestClassInt*",
-        "*IntTestFunction*test_method*",
+        "*Function*test_func*",
+        "*Class*TestClassInt*",
+        "*Function*test_method*",
         "*AppClassCollector*AppTestClass*",
-        "*AppTestMethod*", 
+        "*AppTestMethod*",
     ])
 
 class TestSpaceConfig:
@@ -133,5 +133,5 @@
 
     x = 43
     info = raises(ZeroDivisionError, "x/0")
-    assert info.type is ZeroDivisionError    
-    assert isinstance(info.value, ZeroDivisionError)    
+    assert info.type is ZeroDivisionError
+    assert isinstance(info.value, ZeroDivisionError)
diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -22,15 +22,6 @@
         assert len(failed) == 2
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
-
-    def test_runappdirect(self, testdir):
-        sorter = testdir.inline_run(innertest, '-m', 'applevel -docstring',
-                                    '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
-        assert len(passed) == 4
-        print passed
-        assert "app_test_something" in passed[0].nodeid
-        assert "test_method_app" in passed[1].nodeid
         
     def test_docstring_in_methods(self, testdir): 
         sorter = testdir.inline_run("-k", "AppTestSomething and test_code_in_docstring",

From pypy.commits at gmail.com  Mon Jun 13 11:51:39 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Jun 2016 08:51:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <575ed68b.078e1c0a.5da84.ffffeafe@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85135:fef0c44d0976
Date: 2016-06-13 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/fef0c44d0976/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,9 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -19,3 +22,12 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1203,7 +1203,7 @@
 
     def test_const_fold_unicode_subscr(self, monkeypatch):
         source = """def f():
-        return "abc"[0]
+        return u"abc"[0]
         """
         counts = self.count_instructions(source)
         if 0:   # xxx later?
@@ -1211,14 +1211,14 @@
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
-        return "\U00012345"[0]
+        return u"\U00012345"[0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
         source = """def f():
-        return "\U00012345abcdef"[3]
+        return u"\U00012345abcdef"[3]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
@@ -1226,7 +1226,7 @@
 
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
-        return "\uE01F"[0]
+        return u"\uE01F"[0]
         """
         counts = self.count_instructions(source)
         if 0:   # xxx later?
@@ -1236,7 +1236,7 @@
         # getslice is not yet optimized.
         # Still, check a case which yields the empty string.
         source = """def f():
-        return "abc"[:0]
+        return u"abc"[:0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 3, ops.BUILD_SLICE: 1,
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1516,7 +1516,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -443,7 +443,7 @@
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execv()")
             try:
-                output = "caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             except UnicodeEncodeError:
                 skip("encoding not good enough")
             pid = os.fork()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -23,7 +23,7 @@
             guard_true(i14, descr=...)
             guard_not_invalidated(descr=...)
             i16 = int_eq(i6, %d)
-            i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=)
+            i19 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i6, i10, descr=)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
             i22 = int_ge(i19, i10)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rdynload
 import sys
 
 _WIN = sys.platform == 'win32'
@@ -18,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))',
@@ -79,6 +81,8 @@
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'getdlopenflags'        : 'system.getdlopenflags',
+        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -108,3 +108,9 @@
     ]
     w_thread_info = app.wget(space, "thread_info")
     return space.call_function(w_thread_info, space.newtuple(info_w))
+
+def getdlopenflags(space):
+    return space.wrap(space.sys.dlopenflags)
+
+def setdlopenflags(space, w_flags):
+    space.sys.dlopenflags = space.int_w(w_flags)
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,14 +478,12 @@
 
     def test_dlopenflags(self):
         import sys
-        if hasattr(sys, "setdlopenflags"):
-            assert hasattr(sys, "getdlopenflags")
-            raises(TypeError, sys.getdlopenflags, 42)
-            oldflags = sys.getdlopenflags()
-            raises(TypeError, sys.setdlopenflags)
-            sys.setdlopenflags(oldflags+1)
-            assert sys.getdlopenflags() == oldflags+1
-            sys.setdlopenflags(oldflags)
+        raises(TypeError, sys.getdlopenflags, 42)
+        oldflags = sys.getdlopenflags()
+        raises(TypeError, sys.setdlopenflags)
+        sys.setdlopenflags(oldflags+1)
+        assert sys.getdlopenflags() == oldflags+1
+        sys.setdlopenflags(oldflags)
 
     def test_refcount(self):
         import sys
@@ -661,7 +659,7 @@
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
-        
+
         class Tracer:
             def __init__(self):
                 self.events = []
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -427,11 +427,11 @@
 class FakeModule(W_Root):
     def __init__(self):
         self.w_dict = w_some_obj()
-
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
+FakeObjSpace.sys.dlopenflags = 123
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -181,6 +181,7 @@
             return space.newint(value.count(sub, start, end))
         else:
             res = count(value, sub, start, end)
+            assert res >= 0
             return space.wrap(max(res, 0))
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -218,6 +218,7 @@
 
         check(bytearray(b'abc').replace(b'b', bytearray(b'd')), b'adc')
         check(bytearray(b'abc').replace(b'b', b'd'), b'adc')
+        check(bytearray(b'').replace(b'a', b'ab'), b'')
 
         check(bytearray(b'abc').upper(), b'ABC')
         check(bytearray(b'ABC').lower(), b'abc')
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -3,10 +3,12 @@
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--options] pypy-VER-PLATFORM
+    package.py [--options] --archive-name=pypy-VER-PLATFORM
 
 The output is found in the directory from --builddir,
 by default /tmp/usession-YOURNAME/build/.
+
+For a list of all options, see 'package.py --help'.
 """
 
 import shutil
@@ -64,6 +66,7 @@
     name = options.name
     if not name:
         name = 'pypy-nightly'
+    assert '/' not in name
     rename_pypy_c = options.pypy_c
     override_pypy_c = options.override_pypy_c
 
@@ -299,26 +302,12 @@
         help='destination dir for archive')
     parser.add_argument('--override_pypy_c', type=str, default='',
         help='use as pypy exe instead of pypy/goal/pypy-c')
-    # Positional arguments, for backward compatability with buldbots
-    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
-        metavar='[archive-name] [rename_pypy_c] [targetdir] [override_pypy_c]',
-        )
     options = parser.parse_args(args)
 
-    # Handle positional arguments, choke if both methods are used
-    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
-                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
-        if len(options.extra_args)>i:
-            if getattr(options, target) != default:
-                print 'positional argument',i,target,'already has value',getattr(options, target)
-                parser.print_help()
-                return
-            setattr(options, target, options.extra_args[i])
     if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
         options.nostrip = True
-
     if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        options.tk = True
+        options.no_tk = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -21,8 +21,10 @@
 
     def test_dir_structure(self, test='test'):
         retval, builddir = package.package(
-            '--without-cffi', str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--without-cffi',
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
         assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d' % CPYTHON_VERSION[0]
@@ -71,8 +73,9 @@
         builddir = udir.ensure("build", dir=True)
         retval, builddir = package.package(
             '--without-cffi', '--builddir', str(builddir),
-            str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
 
     def test_with_zipfile_module(self):
         prev = package.USE_ZIPFILE_MODULE
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -324,17 +324,19 @@
         def check(frame):
             expected_size = 1
             idx = 0
+            fixed_size = self.cpu.JITFRAME_FIXED_SIZE
             if self.cpu.backend_name.startswith('arm'):
                 # jitframe fixed part is larger here
                 expected_size = 2
                 idx = 1
+                fixed_size -= 32
             assert len(frame.jf_gcmap) == expected_size
-            if self.cpu.IS_64_BIT:
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE + 1  # +1 from i0
-            else:
-                assert frame.jf_gcmap[idx]
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE - 32 * idx + 1 # +1 from i0
-            assert frame.jf_gcmap[idx] == (1 << (exp_idx + 1)) | (1 << exp_idx)
+            # check that we have two bits set, and that they are in two
+            # registers (p0 and p1 are moved away when doing p2, but not
+            # spilled, just moved to different registers)
+            bits = [n for n in range(fixed_size)
+                      if frame.jf_gcmap[idx] & (1<> (LONG_BIT - 1)) & (p != x))
 
+def _ll_2_int_mod(x, y):
+    # same comments as _ll_2_int_floordiv()
+    r = x % y
+    # the JIT knows that if x and y are both positive, this doesn't change 'r'
+    r -= y & (((x ^ y) & (r | -r)) >> (LONG_BIT - 1))
+    return r
+
 
 def _ll_1_cast_uint_to_float(x):
     # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
@@ -431,6 +438,7 @@
 inline_calls_to = [
     ('int_abs',              [lltype.Signed],                lltype.Signed),
     ('int_floordiv',         [lltype.Signed, lltype.Signed], lltype.Signed),
+    ('int_mod',              [lltype.Signed, lltype.Signed], lltype.Signed),
     ('ll_math.ll_math_sqrt', [lltype.Float],                 lltype.Float),
 ]
 
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -478,7 +478,7 @@
             except ZeroDivisionError:
                 return -42
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_floordiv_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_div_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
@@ -505,7 +505,7 @@
                 return 42
         # XXX so far, this really produces a int_mod_ovf_zer...
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
diff --git a/rpython/jit/codewriter/test/test_support.py b/rpython/jit/codewriter/test/test_support.py
--- a/rpython/jit/codewriter/test/test_support.py
+++ b/rpython/jit/codewriter/test/test_support.py
@@ -144,11 +144,13 @@
     assert _ll_1_int_abs(-10) == 10
     assert _ll_1_int_abs(-sys.maxint) == sys.maxint
 
-def test_int_floordiv():
+def test_int_floordiv_mod():
     from rpython.rtyper.lltypesystem.lloperation import llop
-    from rpython.jit.codewriter.support import _ll_2_int_floordiv
+    from rpython.jit.codewriter.support import _ll_2_int_floordiv, _ll_2_int_mod
     for x in range(-6, 7):
         for y in range(-3, 4):
             if y != 0:
                 assert (_ll_2_int_floordiv(x, y) ==
                         llop.int_floordiv(lltype.Signed, x, y))
+                assert (_ll_2_int_mod(x, y) ==
+                        llop.int_mod(lltype.Signed, x, y))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -97,17 +97,14 @@
         self.emit_operation(op)
 
         r = self.getintbound(op)
-        if b2.is_constant():
-            val = b2.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.is_constant():
-            val = b1.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)):
-            lesser = min(b1.upper, b2.upper)
-            r.intersect(IntBound(0, next_pow2_m1(lesser)))
+        pos1 = b1.known_ge(IntBound(0, 0))
+        pos2 = b2.known_ge(IntBound(0, 0))
+        if pos1 or pos2:
+            r.make_ge(IntBound(0, 0))
+        if pos1:
+            r.make_le(b1)
+        if pos2:
+            r.make_le(b2)
 
     def optimize_INT_SUB(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5188,6 +5188,25 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_and_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        i4 = int_ge(i3, 0)
+        guard_true(i4) []
+        jump(i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        jump(i3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_cmp_above_bounds(self):
         ops = """
         [p0,p1]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -972,6 +972,58 @@
         assert res == expected
         # should contain a call_i(..., OS=OS_INT_PY_DIV)
 
+    def test_int_c_mod(self):
+        from rpython.rlib.rarithmetic import int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_mod(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_MOD)
+
+    def test_positive_c_div_mod(self):
+        from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                assert i > 0
+                t += int_c_div(100, i) - int_c_mod(100, i)
+                i += 1
+            return t
+        expected = sum([100 // n - 100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # all the correction code should be dead now, xxx test that
+
+    def test_int_c_div_by_constant(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = ['k'], reds = ['i', 't'])
+        def f(i, k):
+            t = 0
+            while i < 100:
+                myjitdriver.can_enter_jit(i=i, t=t, k=k)
+                myjitdriver.jit_merge_point(i=i, t=t, k=k)
+                t += int_c_div(i, k)
+                i += 1
+            return t
+        expected = sum([i // 10 for i in range(51, 100)])
+        assert f(-50, 10) == expected
+        res = self.meta_interp(f, [-50, 10])
+        assert res == expected
+        self.check_resops(call=0, uint_mul_high=2)
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -281,11 +281,12 @@
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
+        "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
-        
+
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
@@ -644,6 +645,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -703,6 +705,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -1139,7 +1142,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj),
+                  "Can't forward an object outside the nursery.")
         tid = self.header(obj).tid
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1463,7 +1467,8 @@
                 objhdr.tid |= GCFLAG_CARDS_SET
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1513,7 +1518,8 @@
             return True
         # ^^^ a fast path of write-barrier
         #
-        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+        if (self.card_page_indices > 0 and     # check constant-folded
+            source_hdr.tid & GCFLAG_HAS_CARDS != 0):
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1548,7 +1554,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1721,12 +1728,15 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(
+            self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "pinned_objects_in_nursery != surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(
+                cur >= prev, "pinned objects encountered in backwards order")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1784,7 +1794,8 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
+                  "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -554,6 +554,7 @@
         assert res # we optimized it
         assert hdr_dst.tid & minimark.GCFLAG_TRACK_YOUNG_PTRS == 0 # and we copied the flag
         #
+        self.gc.card_page_indices = 128     # force > 0
         hdr_src.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_dst.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_src.tid |= minimark.GCFLAG_HAS_CARDS
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -148,7 +148,8 @@
                                                  ('elements', FFI_TYPE_PP)])
 
     ffi_cif = rffi_platform.Struct('ffi_cif', [])
-    ffi_closure = rffi_platform.Struct('ffi_closure', [])
+    ffi_closure = rffi_platform.Struct('ffi_closure',
+                                       [('user_data', rffi.VOIDP)])
 
 def add_simple_type(type_name):
     for name in ['size', 'alignment', 'type']:
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -662,6 +662,14 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_floordiv(lltype.Signed, x, y)
 
+def int_c_mod(x, y):
+    """Return the result of the C-style 'x % y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_mod(lltype.Signed, x, y)
+
 @objectmodel.specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -26,7 +26,7 @@
 
 if _MAC_OS:
     pre_include_bits = ['#define MACOSX']
-else: 
+else:
     pre_include_bits = []
 
 if _FREEBSD or _NETBSD or _WIN32:
@@ -145,15 +145,20 @@
         else:
             return lltype.nullptr(rffi.VOIDP.TO)
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        mode = RTLD_NOW
+        if RTLD_LOCAL is not None:
+            mode |= RTLD_LOCAL
+        return mode
+
     def dlopen(name, mode=-1):
         """ Wrapper around C-level dlopen
         """
         if mode == -1:
-            if RTLD_LOCAL is not None:
-                mode = RTLD_LOCAL
-            else:
-                mode = 0
-        if (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
+            mode = _dlopen_default_mode()
+        elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
@@ -193,6 +198,11 @@
     DLLHANDLE = rwin32.HMODULE
     RTLD_GLOBAL = None
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        return 0
+
     def dlopen(name, mode=-1):
         # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -291,6 +291,7 @@
     return _search(value, other, start, end, SEARCH_COUNT)
 
 # -------------- substring searching helper ----------------
+# XXX a lot of code duplication with lltypesystem.rstr :-(
 
 SEARCH_COUNT = 0
 SEARCH_FIND = 1
@@ -309,6 +310,8 @@
     if end > len(value):
         end = len(value)
     if start > end:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     count = 0
@@ -326,6 +329,8 @@
     w = n - m
 
     if w < 0:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     mlast = m - 1
@@ -570,18 +575,20 @@
 
 class ByteListBuilder(object):
     def __init__(self, init_size=INIT_SIZE):
+        assert init_size >= 0
         self.l = newlist_hint(init_size)
 
     @specialize.argtype(1)
     def append(self, s):
+        l = self.l
         for c in s:
-            self.l.append(c)
+            l.append(c)
 
     @specialize.argtype(1)
     def append_slice(self, s, start, end):
-        assert 0 <= start <= end <= len(s)
-        for c in s[start:end]:
-            self.l.append(c)
+        l = self.l
+        for i in xrange(start, end):
+            l.append(s[i])
 
     def append_multiple_char(self, c, times):
         assert isinstance(c, str)
@@ -589,8 +596,9 @@
 
     def append_charpsize(self, s, size):
         assert size >= 0
+        l = self.l
         for i in xrange(size):
-            self.l.append(s[i])
+            l.append(s[i])
 
     def build(self):
         return self.l
diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,17 @@
-#define HAVE_SYS_UCONTEXT_H
+#if !defined(__OpenBSD__)
+#  define HAVE_SYS_UCONTEXT_H
+#else
+#  define HAVE_SIGNAL_H
+#endif
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,10 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(HAVE_SIGNAL_H)
+#include 
+#else
+#  error "don't know how to get the pc on this platform"
 #endif
 
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -401,10 +401,13 @@
 
 @given(strategies.integers(min_value=0, max_value=sys.maxint),
        strategies.integers(min_value=1, max_value=sys.maxint))
-def test_int_c_div(x, y):
+def test_int_c_div_mod(x, y):
     assert int_c_div(~x, y) == -(abs(~x) // y)
     assert int_c_div( x,-y) == -(x // y)
     assert int_c_div(~x,-y) == +(abs(~x) // y)
+    for x1 in [x, ~x]:
+        for y1 in [y, -y]:
+            assert int_c_div(x1, y1) * y1 + int_c_mod(x1, y1) == x1
 
 # these can't be prebuilt on 32bit
 U1 = r_ulonglong(0x0102030405060708L)
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -231,6 +231,10 @@
     check_search(count, 'one two three', 'e', 0, 1, res=0)
     check_search(count, 'one two three', '', 0, 13, res=14)
 
+    check_search(count, '', 'ab', 0, 0, res=0)
+    check_search(count, 'a', 'ab', 0, 1, res=0)
+    check_search(count, 'ac', 'ab', 0, 2, res=0)
+
 
 class TestTranslates(BaseRtypingTest):
     def test_split_rsplit(self):
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -348,19 +348,30 @@
     _about_ = llhelper
 
     def compute_result_annotation(self, s_F, s_callable):
+        from rpython.annotator.description import FunctionDesc
         assert s_F.is_constant()
-        assert s_callable.is_constant()
+        assert isinstance(s_callable, annmodel.SomePBC)
         F = s_F.const
         FUNC = F.TO
         args_s = [lltype_to_annotation(T) for T in FUNC.ARGS]
-        key = (llhelper, s_callable.const)
-        s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
-        assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
+        for desc in s_callable.descriptions:
+            assert isinstance(desc, FunctionDesc)
+            assert desc.pyobj is not None
+            if s_callable.is_constant():
+                assert s_callable.const is desc.pyobj
+            key = (llhelper, desc.pyobj)
+            s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
+            assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
         return SomePtr(F)
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
-        return hop.args_r[1].get_unique_llfn()
+        if hop.args_s[1].is_constant():
+            return hop.args_r[1].get_unique_llfn()
+        else:
+            F = hop.args_s[0].const
+            assert hop.args_r[1].lowleveltype == F
+            return hop.inputarg(hop.args_r[1], 1)
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -236,11 +236,11 @@
         return _rtype_template(hop, 'mul_ovf')
 
     def rtype_floordiv(_, hop):
-        return _rtype_call_helper(hop, 'floordiv', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div', [ZeroDivisionError])
     rtype_inplace_floordiv = rtype_floordiv
 
     def rtype_floordiv_ovf(_, hop):
-        return _rtype_call_helper(hop, 'floordiv_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div_ovf', [ZeroDivisionError])
 
     # turn 'div' on integers into 'floordiv'
     rtype_div         = rtype_floordiv
@@ -250,11 +250,11 @@
     # 'def rtype_truediv' is delegated to the superclass FloatRepr
 
     def rtype_mod(_, hop):
-        return _rtype_call_helper(hop, 'mod', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod', [ZeroDivisionError])
     rtype_inplace_mod = rtype_mod
 
     def rtype_mod_ovf(_, hop):
-        return _rtype_call_helper(hop, 'mod_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod_ovf', [ZeroDivisionError])
 
     def rtype_xor(_, hop):
         return _rtype_template(hop, 'xor')
@@ -319,7 +319,7 @@
     vlist = hop.inputargs(repr, repr2)
     prefix = repr.opprefix
 
-    if '_ovf' in func or func.startswith(('mod', 'floordiv')):
+    if '_ovf' in func or func.startswith(('py_mod', 'py_div')):
         if prefix+func not in ('int_add_ovf', 'int_add_nonneg_ovf',
                                'int_sub_ovf', 'int_mul_ovf'):
             raise TyperError("%r should not be used here any more" % (func,))
@@ -353,7 +353,7 @@
             any_implicit_exception = True
 
     if not any_implicit_exception:
-        if not func.startswith(('mod', 'floordiv')):
+        if not func.startswith(('py_mod', 'py_div')):
             return _rtype_template(hop, func)
 
     repr = hop.r_result
@@ -388,7 +388,7 @@
 # ---------- floordiv ----------
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv(x, y):
+def ll_int_py_div(x, y):
     # Python, and RPython, assume that integer division truncates
     # towards -infinity.  However, in C, integer division truncates
     # towards 0.  So assuming that, we need to apply a correction
@@ -400,159 +400,159 @@
     return r + (u >> INT_BITS_1)
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv_nonnegargs(x, y):
+def ll_int_py_div_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_floordiv(Signed, x, y)            # <= truncates like in C
-    ll_assert(r >= 0, "int_floordiv_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_div_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_floordiv_zer(x, y):
+def ll_int_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf(x, y):
+def ll_int_py_div_ovf(x, y):
     # JIT: intentionally not short-circuited to produce only one guard
     # and to remove the check fully if one of the arguments is known
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf_zer(x, y):
+def ll_int_py_div_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv_ovf(x, y)
+    return ll_int_py_div_ovf(x, y)
 
 @jit.oopspec("int.udiv(x, y)")
-def ll_uint_floordiv(x, y):
+def ll_uint_py_div(x, y):
     return llop.uint_floordiv(Unsigned, x, y)
 
-def ll_uint_floordiv_zer(x, y):
+def ll_uint_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("unsigned integer division")
-    return ll_uint_floordiv(x, y)
+    return ll_uint_py_div(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_floordiv      = ll_int_floordiv
-    ll_llong_floordiv_zer  = ll_int_floordiv_zer
-    ll_ullong_floordiv     = ll_uint_floordiv
-    ll_ullong_floordiv_zer = ll_uint_floordiv_zer
+    ll_llong_py_div      = ll_int_py_div
+    ll_llong_py_div_zer  = ll_int_py_div_zer
+    ll_ullong_py_div     = ll_uint_py_div
+    ll_ullong_py_div_zer = ll_uint_py_div_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_floordiv(x, y):
+    def ll_llong_py_div(x, y):
         r = llop.llong_floordiv(SignedLongLong, x, y)  # <= truncates like in C
         p = r * y
         if y < 0: u = p - x
         else:     u = x - p
         return r + (u >> LLONG_BITS_1)
 
-    def ll_llong_floordiv_zer(x, y):
+    def ll_llong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("longlong division")
-        return ll_llong_floordiv(x, y)
+        return ll_llong_py_div(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_floordiv(x, y):
+    def ll_ullong_py_div(x, y):
         return llop.ullong_floordiv(UnsignedLongLong, x, y)
 
-    def ll_ullong_floordiv_zer(x, y):
+    def ll_ullong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("unsigned longlong division")
-        return ll_ullong_floordiv(x, y)
+        return ll_ullong_py_div(x, y)
 
 @jit.dont_look_inside
-def ll_lllong_floordiv(x, y):
+def ll_lllong_py_div(x, y):
     r = llop.lllong_floordiv(SignedLongLongLong, x, y) # <= truncates like in C
     p = r * y
     if y < 0: u = p - x
     else:     u = x - p
     return r + (u >> LLLONG_BITS_1)
 
-def ll_lllong_floordiv_zer(x, y):
+def ll_lllong_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("longlonglong division")
-    return ll_lllong_floordiv(x, y)
+    return ll_lllong_py_div(x, y)
 
 
 # ---------- mod ----------
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod(x, y):
+def ll_int_py_mod(x, y):
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> INT_BITS_1))
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod_nonnegargs(x, y):
+def ll_int_py_mod_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
-    ll_assert(r >= 0, "int_mod_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_mod_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_mod_zer(x, y):
+def ll_int_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf(x, y):
-    # see comment in ll_int_floordiv_ovf
+def ll_int_py_mod_ovf(x, y):
+    # see comment in ll_int_py_div_ovf
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf_zer(x, y):
+def ll_int_py_mod_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod_ovf(x, y)
+    return ll_int_py_mod_ovf(x, y)
 
 @jit.oopspec("int.umod(x, y)")
-def ll_uint_mod(x, y):
+def ll_uint_py_mod(x, y):
     return llop.uint_mod(Unsigned, x, y)
 
-def ll_uint_mod_zer(x, y):
+def ll_uint_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_uint_mod(x, y)
+    return ll_uint_py_mod(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_mod      = ll_int_mod
-    ll_llong_mod_zer  = ll_int_mod_zer
-    ll_ullong_mod     = ll_uint_mod
-    ll_ullong_mod_zer = ll_uint_mod_zer
+    ll_llong_py_mod      = ll_int_py_mod
+    ll_llong_py_mod_zer  = ll_int_py_mod_zer
+    ll_ullong_py_mod     = ll_uint_py_mod
+    ll_ullong_py_mod_zer = ll_uint_py_mod_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_mod(x, y):
+    def ll_llong_py_mod(x, y):
         r = llop.llong_mod(SignedLongLong, x, y)    # <= truncates like in C
         if y < 0: u = -r
         else:     u = r
         return r + (y & (u >> LLONG_BITS_1))
 
-    def ll_llong_mod_zer(x, y):
+    def ll_llong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return ll_llong_mod(x, y)
+        return ll_llong_py_mod(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_mod(x, y):
+    def ll_ullong_py_mod(x, y):
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
-    def ll_ullong_mod_zer(x, y):
+    def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
 @jit.dont_look_inside
-def ll_lllong_mod(x, y):
+def ll_lllong_py_mod(x, y):
     r = llop.lllong_mod(SignedLongLongLong, x, y)  # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> LLLONG_BITS_1))
 
-def ll_lllong_mod_zer(x, y):
+def ll_lllong_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_lllong_mod(x, y)
+    return ll_lllong_py_mod(x, y)
 
 
 # ---------- lshift, neg, abs ----------
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -414,7 +414,7 @@
         if self.s_pbc.can_be_None:
             self.descriptions.insert(0, None)
         POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
-                              hints={'nolength': True})
+                              hints={'nolength': True, 'immutable': True})
         pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
                                immortal=True)
         for i, desc in enumerate(self.descriptions):
@@ -564,7 +564,7 @@
     if r_to in r_from._conversion_tables:
         return r_from._conversion_tables[r_to]
     else:
-        t = malloc(Array(Char, hints={'nolength': True}),
+        t = malloc(Array(Char, hints={'nolength': True, 'immutable': True}),
                    len(r_from.descriptions), immortal=True)
         l = []
         for i, d in enumerate(r_from.descriptions):
@@ -577,7 +577,7 @@
         if l == range(len(r_from.descriptions)):
             r = None
         else:
-            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
+            r = inputconst(typeOf(t), t)
         r_from._conversion_tables[r_to] = r
         return r
 
diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -462,6 +462,30 @@
     res = interpret(h, [8, 5, 2])
     assert res == 99
 
+def test_llhelper_multiple_functions():
+    S = GcStruct('S', ('x', Signed), ('y', Signed))
+    def f(s):
+        return s.x - s.y
+    def g(s):
+        return s.x + s.y
+
+    F = Ptr(FuncType([Ptr(S)], Signed))
+
+    myfuncs = [f, g]
+
+    def h(x, y, z):
+        s = malloc(S)
+        s.x = x
+        s.y = y
+        fptr = llhelper(F, myfuncs[z])
+        assert typeOf(fptr) == F
+        return fptr(s)
+
+    res = interpret(h, [80, 5, 0])
+    assert res == 75
+    res = interpret(h, [80, 5, 1])
+    assert res == 85
+
 
 def test_cast_instance_to_base_ptr():
     class A:
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -390,7 +390,7 @@
         res = self.interpret(f, [sys.maxint])
         assert res == 0
 
-    def test_int_floordiv_nonnegargs(self):
+    def test_int_py_div_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
@@ -398,7 +398,7 @@
         res = self.interpret(f, [1234567, 123])
         assert res == 1234567 // 123
 
-    def test_int_mod_nonnegargs(self):
+    def test_int_py_mod_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -2012,6 +2012,36 @@
         e = py.test.raises(ValueError, self.interpret, f, [3])
         assert str(e.value).startswith(r"exit case '\xff' not found")
 
+    @py.test.mark.parametrize('limit', [3, 5])
+    def test_conversion_table(self, limit):
+        # with limit==3, check conversion from Char to Ptr(Func).
+        # with limit>3, check conversion from Char to Char.
+        def f1():
+            return 111
+        def f2():
+            return 222
+        def f3():
+            return 333
+        def g(n):
+            if n & 1:
+                return f1
+            else:
+                return f2
+        def f(n):
+            x = g(n)    # can be f1 or f2
+            if n > 10:
+                x = f3  # now can be f1 or f2 or f3
+            return x()
+
+        from rpython.config.translationoption import get_combined_translation_config
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.withsmallfuncsets = limit
+        assert self.interpret(f, [3]) == 111
+        assert self.interpret(f, [4]) == 222
+        assert self.interpret(f, [13]) == 333
+        assert self.interpret(f, [14]) == 333
+
+
 def test_smallfuncsets_basic():
     from rpython.translator.translator import TranslationContext, graphof
     from rpython.config.translationoption import get_combined_translation_config
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -972,6 +972,13 @@
             s.count(s, -10)
         py.test.raises(AnnotatorError, self.interpret, f, ())
 
+    def test_count_in_empty_string(self):
+        const = self.const
+        def fn():
+            return const('').count(const('ab'))
+        res = self.interpret(fn, [])
+        assert res == 0
+
     def test_getitem_exc(self):
         const = self.const
         def f(x):
diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py
--- a/rpython/translator/sandbox/sandlib.py
+++ b/rpython/translator/sandbox/sandlib.py
@@ -530,6 +530,9 @@
     def do_ll_os__ll_os_unlink(self, vpathname):
         raise OSError(errno.EPERM, "write access denied")
 
+    def do_ll_os__ll_os_mkdir(self, vpathname, mode=None):
+        raise OSError(errno.EPERM, "write access denied")
+
     def do_ll_os__ll_os_getuid(self):
         return UID
     do_ll_os__ll_os_geteuid = do_ll_os__ll_os_getuid
diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py
--- a/rpython/translator/test/test_simplify.py
+++ b/rpython/translator/test/test_simplify.py
@@ -55,7 +55,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \
@@ -73,7 +73,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \

From pypy.commits at gmail.com  Mon Jun 13 12:48:35 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 13 Jun 2016 09:48:35 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix merge
Message-ID: <575ee3e3.4f941c0a.d84c7.6171@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85136:de092081d81f
Date: 2016-06-13 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/de092081d81f/

Log:	fix merge

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -241,7 +241,7 @@
         self._version_tag = version_tag
         self.terminator.mutated_w_cls_version(version_tag)
 
-    def getattribute_if_not_from_object(w_self):
+    def getattribute_if_not_from_object(self):
         """ this method returns the applevel __getattribute__ if that is not
         the one from object, in which case it returns None """
         from pypy.objspace.descroperation import object_getattribute

From pypy.commits at gmail.com  Mon Jun 13 13:30:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 10:30:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Improve CPython compatibility of 'is'.
 Now 'x is y' is guaranteed
Message-ID: <575eed9e.c255c20a.588fa.0e26@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85137:418b05f95db5
Date: 2016-06-13 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/418b05f95db5/

Log:	Improve CPython compatibility of 'is'. Now 'x is y' is guaranteed to
	return True if x == y and x, y are:

	 * empty strings; empty unicode strings

	 * single-character (unicode) strings

	 * empty tuples

	This is in addition to all other special cases (ints, etc.)

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.unicodeobject import (
     decode_object, unicode_from_encoded_object,
     unicode_from_string, getdefaultencoding)
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
 class W_AbstractBytesObject(W_Root):
@@ -30,12 +31,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.str_w(self) is space.str_w(w_other)
+        s1 = space.str_w(self)
+        s2 = space.str_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.str_w(self)))
+        s = space.str_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ord(s[0])     # base values 0-255
+            else:
+                base = 256           # empty string: base value 256
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def unicode_w(self, space):
         # Use the default encoding.
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -186,17 +186,28 @@
     def test_id_on_strs(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        u = u"a"
-        assert id(self.unwrap_wrap_unicode(u)) == id(u)
-        s = "a"
-        assert id(self.unwrap_wrap_str(s)) == id(s)
+        for u in [u"", u"a", u"aa"]:
+            assert id(self.unwrap_wrap_unicode(u)) == id(u)
+            s = str(u)
+            assert id(self.unwrap_wrap_str(s)) == id(s)
+        #
+        assert id('') == (256 << 4) | 11     # always
+        assert id(u'') == (257 << 4) | 11
+        assert id('a') == (ord('a') << 4) | 11
+        assert id(u'\u1234') == ((~0x1234) << 4) | 11
+
+    def test_id_of_tuples(self):
+        l = []
+        x = (l,)
+        assert id(x) != id((l,))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(()) == (258 << 4) | 11     # always
 
     def test_identity_vs_id_primitives(self):
-        if self.cpython_apptest:
-            skip("cpython behaves differently")
         import sys
-        l = range(-10, 10)
-        for i in range(10):
+        l = range(-10, 10, 2)
+        for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
             l.append(long(i))
@@ -206,18 +217,13 @@
             l.append(i - 1j)
             l.append(1 + i * 1j)
             l.append(1 - i * 1j)
-            s = str(i)
-            l.append(s)
-            u = unicode(s)
-            l.append(u)
+            l.append((i,))
         l.append(-0.0)
         l.append(None)
         l.append(True)
         l.append(False)
-        s = "s"
-        l.append(s)
-        s = u"s"
-        l.append(s)
+        l.append(())
+        l.append(tuple([]))
 
         for i, a in enumerate(l):
             for b in l[i:]:
@@ -228,21 +234,18 @@
     def test_identity_vs_id_str(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        import sys
-        l = range(-10, 10)
-        for i in range(10):
-            s = str(i)
+        l = []
+        def add(s, u):
             l.append(s)
             l.append(self.unwrap_wrap_str(s))
-            u = unicode(s)
+            l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
-        s = "s"
-        l.append(s)
-        l.append(self.unwrap_wrap_str(s))
-        s = u"s"
-        l.append(s)
-        l.append(self.unwrap_wrap_unicode(s))
+            l.append(u[:1] + u[1:])
+        for i in range(3, 18):
+            add(str(i), unicode(i))
+        add("s", u"s")
+        add("", u"")
 
         for i, a in enumerate(l):
             for b in l[i:]:
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -9,7 +9,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop,
     normalize_simple_slice)
-from pypy.objspace.std.util import negate
+from pypy.objspace.std.util import negate, IDTAG_SPECIAL, IDTAG_SHIFT
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import intmask
@@ -38,6 +38,23 @@
 class W_AbstractTupleObject(W_Root):
     __slots__ = ()
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractTupleObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty tuples are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty tuple: base value 258
+        uid = (258 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def __repr__(self):
         """representation for debugging purposes"""
         reprlist = [repr(w_item) for w_item in self.tolist()]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
@@ -52,12 +53,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.unicode_w(self) is space.unicode_w(w_other)
+        s1 = space.unicode_w(self)
+        s2 = space.unicode_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.unicode_w(self)))
+        s = space.unicode_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ~ord(s[0])      # negative base values
+            else:
+                base = 257       # empty unicode string: base value 257
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def str_w(self, space):
         return space.str_w(space.str(self))
diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py
--- a/pypy/objspace/std/util.py
+++ b/pypy/objspace/std/util.py
@@ -9,6 +9,11 @@
 IDTAG_FLOAT   = 5
 IDTAG_COMPLEX = 7
 IDTAG_METHOD  = 9
+IDTAG_SPECIAL = 11    # -1 - (-maxunicode-1): unichar
+                      # 0 - 255: char
+                      # 256: empty string
+                      # 257: empty unicode
+                      # 258: empty tuple
 
 CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
 BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',

From pypy.commits at gmail.com  Mon Jun 13 13:30:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 10:30:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document the new behavior of 'id/is' on
 str/unicode/tuple.
Message-ID: <575eeda0.832c1c0a.89c46.0990@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85138:dbd7b8e24c1b
Date: 2016-06-13 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/dbd7b8e24c1b/

Log:	Document the new behavior of 'id/is' on str/unicode/tuple.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,26 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``
+and ``tuple`` rules were added in PyPy 5.4; before that, a test like
+``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was equal to
+``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is closer to
+CPython's, which caches precisely the empty string, unicode and tuple,
+and (sometimes!) the single-character strings and unicodes.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,

From pypy.commits at gmail.com  Mon Jun 13 13:41:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 10:41:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add id/is support to the empty
 frozenset, similar to the empty tuple
Message-ID: <575ef032.4e0b1c0a.91a46.1938@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85139:b80452987110
Date: 2016-06-13 19:40 +0200
http://bitbucket.org/pypy/pypy/changeset/b80452987110/

Log:	Add id/is support to the empty frozenset, similar to the empty tuple

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -321,6 +321,8 @@
 
  - ``tuple`` (empty tuples only)
 
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
@@ -329,12 +331,13 @@
 Note that strings of length 2 or greater can be equal without being
 identical.  Similarly, ``x is (2,)`` is not necessarily true even if
 ``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
-only to the particular cases described above.  The ``str``, ``unicode``
-and ``tuple`` rules were added in PyPy 5.4; before that, a test like
-``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was equal to
-``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is closer to
-CPython's, which caches precisely the empty string, unicode and tuple,
-and (sometimes!) the single-character strings and unicodes.
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty
+string/unicode/tuple/frozenset, and (sometimes!) the single-character
+strings and unicodes.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -6,6 +6,7 @@
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 from rpython.rlib.objectmodel import r_dict
 from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash
@@ -575,6 +576,23 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_FrozensetObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty frozensets are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty frozenset: base value 259
+        uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def _newobj(self, space, w_iterable):
         """Make a new frozenset by taking ownership of 'w_iterable'."""
         if type(self) is W_FrozensetObject:
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -204,6 +204,14 @@
             skip("cannot run this test as apptest")
         assert id(()) == (258 << 4) | 11     # always
 
+    def test_id_of_frozensets(self):
+        x = frozenset([4])
+        assert id(x) != id(frozenset([4]))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(frozenset()) == (259 << 4) | 11     # always
+        assert id(frozenset([])) == (259 << 4) | 11   # always
+
     def test_identity_vs_id_primitives(self):
         import sys
         l = range(-10, 10, 2)
@@ -218,12 +226,14 @@
             l.append(1 + i * 1j)
             l.append(1 - i * 1j)
             l.append((i,))
+            l.append(frozenset([i]))
         l.append(-0.0)
         l.append(None)
         l.append(True)
         l.append(False)
         l.append(())
         l.append(tuple([]))
+        l.append(frozenset())
 
         for i, a in enumerate(l):
             for b in l[i:]:
diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py
--- a/pypy/objspace/std/util.py
+++ b/pypy/objspace/std/util.py
@@ -14,6 +14,7 @@
                       # 256: empty string
                       # 257: empty unicode
                       # 258: empty tuple
+                      # 259: empty frozenset
 
 CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
 BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',

From pypy.commits at gmail.com  Mon Jun 13 13:50:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Jun 2016 10:50:53 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: fix test_ndarrayobject: replace
 RPythonism with regular C
Message-ID: <575ef27d.426dc20a.8e816.ffffcc60@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85140:bd32cf80bb35
Date: 2016-06-13 18:49 +0100
http://bitbucket.org/pypy/pypy/changeset/bd32cf80bb35/

Log:	fix test_ndarrayobject: replace RPythonism with regular C

diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.descriptor import get_dtype_cache
-import pypy.module.micronumpy.constants as NPY 
+import pypy.module.micronumpy.constants as NPY
 
 def scalar(space):
     dtype = get_dtype_cache(space).w_float64dtype
@@ -237,7 +237,7 @@
             except:
                 skip('numpy not importable')
         else:
-            numpy_incl = os.path.abspath(os.path.dirname(__file__) + 
+            numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
             assert os.path.exists(numpy_incl)
             cls.w_numpy_include = cls.space.wrap([numpy_incl])
@@ -273,7 +273,7 @@
                 {
                     /* Should have failed */
                     Py_DECREF(obj1);
-                    return NULL; 
+                    return NULL;
                 }
                 return obj1;
                 '''
@@ -300,14 +300,14 @@
                 ),
                 ("test_DescrFromType", "METH_O",
                 """
-                    Signed typenum = PyInt_AsLong(args);
+                    long typenum = PyInt_AsLong(args);
                     return PyArray_DescrFromType(typenum);
                 """
                 ),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -315,7 +315,7 @@
                     #define PyArray_FromObject _PyArray_FromObject
                     #define PyArray_FromAny _PyArray_FromAny
                 #endif
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -349,14 +349,14 @@
                     Py_INCREF(obj);
                     return obj;
                 '''),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -403,14 +403,14 @@
                 void *array_data[] = {NULL, NULL};
                 return PyUFunc_FromFuncAndDataAndSignature(funcs,
                                     array_data, types, 1, 1, 1, PyUFunc_None,
-                                    "float_3x3", 
-                                    "a ufunc that tests a more complicated signature", 
+                                    "float_3x3",
+                                    "a ufunc that tests a more complicated signature",
                                     0, "(m,m)->(m,m)");
                 """),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -480,7 +480,7 @@
                             res += +10;
                     *((float *)args[1]) = res;
                 };
-                            
+
                 ''',  more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();

From pypy.commits at gmail.com  Mon Jun 13 14:24:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 11:24:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Mention this change in whatsnew-head
Message-ID: <575efa47.e976c20a.17417.ffffcedc@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85141:66f63c100c7b
Date: 2016-06-13 20:24 +0200
http://bitbucket.org/pypy/pypy/changeset/66f63c100c7b/

Log:	Mention this change in whatsnew-head

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,10 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
 .. pull request #455
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 

From pypy.commits at gmail.com  Mon Jun 13 15:05:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 12:05:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test (after 4e4fa6046959,
 which gets rid of these unexpected
Message-ID: <575f0400.011f1c0a.5ec00.0b0b@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85142:8030d6268539
Date: 2016-06-13 21:06 +0200
http://bitbucket.org/pypy/pypy/changeset/8030d6268539/

Log:	Fix test (after 4e4fa6046959, which gets rid of these unexpected
	getarrayitem_raw)

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -163,14 +163,10 @@
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
             guard_true(i32, descr=...)
-            i34 = getarrayitem_raw_i(#, #, descr=)  # XXX what are these?
-            guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i35 = getarrayitem_raw_i(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, 8)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, #, descr=...)                  # XXX
             jump(..., descr=...)
         """)
 

From pypy.commits at gmail.com  Mon Jun 13 15:06:03 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 13 Jun 2016 12:06:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove state.ast_type double quotes to
 get old format back
Message-ID: <575f041b.09f6c20a.38379.ffffcc14@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85143:6ed651b758f8
Date: 2016-06-13 21:05 +0200
http://bitbucket.org/pypy/pypy/changeset/6ed651b758f8/

Log:	Remove state.ast_type double quotes to get old format back

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -204,7 +204,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Module(_body)
 
-State.ast_type('Module', 'mod', ["'body'"])
+State.ast_type('Module', 'mod', ['body'])
 
 
 class Interactive(mod):
@@ -237,7 +237,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Interactive(_body)
 
-State.ast_type('Interactive', 'mod', ["'body'"])
+State.ast_type('Interactive', 'mod', ['body'])
 
 
 class Expression(mod):
@@ -266,7 +266,7 @@
             raise_required_value(space, w_node, 'body')
         return Expression(_body)
 
-State.ast_type('Expression', 'mod', ["'body'"])
+State.ast_type('Expression', 'mod', ['body'])
 
 
 class Suite(mod):
@@ -299,7 +299,7 @@
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         return Suite(_body)
 
-State.ast_type('Suite', 'mod', ["'body'"])
+State.ast_type('Suite', 'mod', ['body'])
 
 
 class stmt(AST):
@@ -356,7 +356,7 @@
             return Continue.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected stmt node, got %T", w_node)
-State.ast_type('stmt', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('stmt', 'AST', None, ['lineno', 'col_offset'])
 
 class FunctionDef(stmt):
 
@@ -431,7 +431,7 @@
         _col_offset = space.int_w(w_col_offset)
         return FunctionDef(_name, _args, _body, _decorator_list, _returns, _lineno, _col_offset)
 
-State.ast_type('FunctionDef', 'stmt', ["'name'", "'args'", "'body'", "'decorator_list'", "'returns'"])
+State.ast_type('FunctionDef', 'stmt', ['name', 'args', 'body', 'decorator_list', 'returns'])
 
 
 class ClassDef(stmt):
@@ -516,7 +516,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ClassDef(_name, _bases, _keywords, _body, _decorator_list, _lineno, _col_offset)
 
-State.ast_type('ClassDef', 'stmt', ["'name'", "'bases'", "'keywords'", "'body'", "'decorator_list'"])
+State.ast_type('ClassDef', 'stmt', ['name', 'bases', 'keywords', 'body', 'decorator_list'])
 
 
 class Return(stmt):
@@ -553,7 +553,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Return(_value, _lineno, _col_offset)
 
-State.ast_type('Return', 'stmt', ["'value'"])
+State.ast_type('Return', 'stmt', ['value'])
 
 
 class Delete(stmt):
@@ -595,7 +595,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Delete(_targets, _lineno, _col_offset)
 
-State.ast_type('Delete', 'stmt', ["'targets'"])
+State.ast_type('Delete', 'stmt', ['targets'])
 
 
 class Assign(stmt):
@@ -645,7 +645,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assign(_targets, _value, _lineno, _col_offset)
 
-State.ast_type('Assign', 'stmt', ["'targets'", "'value'"])
+State.ast_type('Assign', 'stmt', ['targets', 'value'])
 
 
 class AugAssign(stmt):
@@ -698,7 +698,7 @@
         _col_offset = space.int_w(w_col_offset)
         return AugAssign(_target, _op, _value, _lineno, _col_offset)
 
-State.ast_type('AugAssign', 'stmt', ["'target'", "'op'", "'value'"])
+State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value'])
 
 
 class For(stmt):
@@ -768,7 +768,7 @@
         _col_offset = space.int_w(w_col_offset)
         return For(_target, _iter, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('For', 'stmt', ["'target'", "'iter'", "'body'", "'orelse'"])
+State.ast_type('For', 'stmt', ['target', 'iter', 'body', 'orelse'])
 
 
 class While(stmt):
@@ -830,7 +830,7 @@
         _col_offset = space.int_w(w_col_offset)
         return While(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('While', 'stmt', ["'test'", "'body'", "'orelse'"])
+State.ast_type('While', 'stmt', ['test', 'body', 'orelse'])
 
 
 class If(stmt):
@@ -892,7 +892,7 @@
         _col_offset = space.int_w(w_col_offset)
         return If(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('If', 'stmt', ["'test'", "'body'", "'orelse'"])
+State.ast_type('If', 'stmt', ['test', 'body', 'orelse'])
 
 
 class With(stmt):
@@ -946,7 +946,7 @@
         _col_offset = space.int_w(w_col_offset)
         return With(_items, _body, _lineno, _col_offset)
 
-State.ast_type('With', 'stmt', ["'items'", "'body'"])
+State.ast_type('With', 'stmt', ['items', 'body'])
 
 
 class Raise(stmt):
@@ -990,7 +990,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Raise(_exc, _cause, _lineno, _col_offset)
 
-State.ast_type('Raise', 'stmt', ["'exc'", "'cause'"])
+State.ast_type('Raise', 'stmt', ['exc', 'cause'])
 
 
 class Try(stmt):
@@ -1068,7 +1068,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Try(_body, _handlers, _orelse, _finalbody, _lineno, _col_offset)
 
-State.ast_type('Try', 'stmt', ["'body'", "'handlers'", "'orelse'", "'finalbody'"])
+State.ast_type('Try', 'stmt', ['body', 'handlers', 'orelse', 'finalbody'])
 
 
 class Assert(stmt):
@@ -1113,7 +1113,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Assert(_test, _msg, _lineno, _col_offset)
 
-State.ast_type('Assert', 'stmt', ["'test'", "'msg'"])
+State.ast_type('Assert', 'stmt', ['test', 'msg'])
 
 
 class Import(stmt):
@@ -1155,7 +1155,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Import(_names, _lineno, _col_offset)
 
-State.ast_type('Import', 'stmt', ["'names'"])
+State.ast_type('Import', 'stmt', ['names'])
 
 
 class ImportFrom(stmt):
@@ -1207,7 +1207,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ImportFrom(_module, _names, _level, _lineno, _col_offset)
 
-State.ast_type('ImportFrom', 'stmt', ["'module'", "'names'", "'level'"])
+State.ast_type('ImportFrom', 'stmt', ['module', 'names', 'level'])
 
 
 class Global(stmt):
@@ -1247,7 +1247,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
 
-State.ast_type('Global', 'stmt', ["'names'"])
+State.ast_type('Global', 'stmt', ['names'])
 
 
 class Nonlocal(stmt):
@@ -1287,7 +1287,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Nonlocal(_names, _lineno, _col_offset)
 
-State.ast_type('Nonlocal', 'stmt', ["'names'"])
+State.ast_type('Nonlocal', 'stmt', ['names'])
 
 
 class Expr(stmt):
@@ -1325,7 +1325,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Expr(_value, _lineno, _col_offset)
 
-State.ast_type('Expr', 'stmt', ["'value'"])
+State.ast_type('Expr', 'stmt', ['value'])
 
 
 class Pass(stmt):
@@ -1484,7 +1484,7 @@
             return Const.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected expr node, got %T", w_node)
-State.ast_type('expr', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('expr', 'AST', None, ['lineno', 'col_offset'])
 
 class BoolOp(expr):
 
@@ -1532,7 +1532,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BoolOp(_op, _values, _lineno, _col_offset)
 
-State.ast_type('BoolOp', 'expr', ["'op'", "'values'"])
+State.ast_type('BoolOp', 'expr', ['op', 'values'])
 
 
 class BinOp(expr):
@@ -1585,7 +1585,7 @@
         _col_offset = space.int_w(w_col_offset)
         return BinOp(_left, _op, _right, _lineno, _col_offset)
 
-State.ast_type('BinOp', 'expr', ["'left'", "'op'", "'right'"])
+State.ast_type('BinOp', 'expr', ['left', 'op', 'right'])
 
 
 class UnaryOp(expr):
@@ -1630,7 +1630,7 @@
         _col_offset = space.int_w(w_col_offset)
         return UnaryOp(_op, _operand, _lineno, _col_offset)
 
-State.ast_type('UnaryOp', 'expr', ["'op'", "'operand'"])
+State.ast_type('UnaryOp', 'expr', ['op', 'operand'])
 
 
 class Lambda(expr):
@@ -1676,7 +1676,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Lambda(_args, _body, _lineno, _col_offset)
 
-State.ast_type('Lambda', 'expr', ["'args'", "'body'"])
+State.ast_type('Lambda', 'expr', ['args', 'body'])
 
 
 class IfExp(expr):
@@ -1730,7 +1730,7 @@
         _col_offset = space.int_w(w_col_offset)
         return IfExp(_test, _body, _orelse, _lineno, _col_offset)
 
-State.ast_type('IfExp', 'expr', ["'test'", "'body'", "'orelse'"])
+State.ast_type('IfExp', 'expr', ['test', 'body', 'orelse'])
 
 
 class Dict(expr):
@@ -1784,7 +1784,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Dict(_keys, _values, _lineno, _col_offset)
 
-State.ast_type('Dict', 'expr', ["'keys'", "'values'"])
+State.ast_type('Dict', 'expr', ['keys', 'values'])
 
 
 class Set(expr):
@@ -1826,7 +1826,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Set(_elts, _lineno, _col_offset)
 
-State.ast_type('Set', 'expr', ["'elts'"])
+State.ast_type('Set', 'expr', ['elts'])
 
 
 class ListComp(expr):
@@ -1876,7 +1876,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ListComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('ListComp', 'expr', ["'elt'", "'generators'"])
+State.ast_type('ListComp', 'expr', ['elt', 'generators'])
 
 
 class SetComp(expr):
@@ -1926,7 +1926,7 @@
         _col_offset = space.int_w(w_col_offset)
         return SetComp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('SetComp', 'expr', ["'elt'", "'generators'"])
+State.ast_type('SetComp', 'expr', ['elt', 'generators'])
 
 
 class DictComp(expr):
@@ -1984,7 +1984,7 @@
         _col_offset = space.int_w(w_col_offset)
         return DictComp(_key, _value, _generators, _lineno, _col_offset)
 
-State.ast_type('DictComp', 'expr', ["'key'", "'value'", "'generators'"])
+State.ast_type('DictComp', 'expr', ['key', 'value', 'generators'])
 
 
 class GeneratorExp(expr):
@@ -2034,7 +2034,7 @@
         _col_offset = space.int_w(w_col_offset)
         return GeneratorExp(_elt, _generators, _lineno, _col_offset)
 
-State.ast_type('GeneratorExp', 'expr', ["'elt'", "'generators'"])
+State.ast_type('GeneratorExp', 'expr', ['elt', 'generators'])
 
 
 class Yield(expr):
@@ -2071,7 +2071,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Yield(_value, _lineno, _col_offset)
 
-State.ast_type('Yield', 'expr', ["'value'"])
+State.ast_type('Yield', 'expr', ['value'])
 
 
 class YieldFrom(expr):
@@ -2109,7 +2109,7 @@
         _col_offset = space.int_w(w_col_offset)
         return YieldFrom(_value, _lineno, _col_offset)
 
-State.ast_type('YieldFrom', 'expr', ["'value'"])
+State.ast_type('YieldFrom', 'expr', ['value'])
 
 
 class Compare(expr):
@@ -2169,7 +2169,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Compare(_left, _ops, _comparators, _lineno, _col_offset)
 
-State.ast_type('Compare', 'expr', ["'left'", "'ops'", "'comparators'"])
+State.ast_type('Compare', 'expr', ['left', 'ops', 'comparators'])
 
 
 class Call(expr):
@@ -2231,7 +2231,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Call(_func, _args, _keywords, _lineno, _col_offset)
 
-State.ast_type('Call', 'expr', ["'func'", "'args'", "'keywords'"])
+State.ast_type('Call', 'expr', ['func', 'args', 'keywords'])
 
 
 class Num(expr):
@@ -2268,7 +2268,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Num(_n, _lineno, _col_offset)
 
-State.ast_type('Num', 'expr', ["'n'"])
+State.ast_type('Num', 'expr', ['n'])
 
 
 class Str(expr):
@@ -2305,7 +2305,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Str(_s, _lineno, _col_offset)
 
-State.ast_type('Str', 'expr', ["'s'"])
+State.ast_type('Str', 'expr', ['s'])
 
 
 class Bytes(expr):
@@ -2342,7 +2342,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Bytes(_s, _lineno, _col_offset)
 
-State.ast_type('Bytes', 'expr', ["'s'"])
+State.ast_type('Bytes', 'expr', ['s'])
 
 
 class NameConstant(expr):
@@ -2379,7 +2379,7 @@
         _col_offset = space.int_w(w_col_offset)
         return NameConstant(_value, _lineno, _col_offset)
 
-State.ast_type('NameConstant', 'expr', ["'value'"])
+State.ast_type('NameConstant', 'expr', ['value'])
 
 
 class Ellipsis(expr):
@@ -2461,7 +2461,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Attribute(_value, _attr, _ctx, _lineno, _col_offset)
 
-State.ast_type('Attribute', 'expr', ["'value'", "'attr'", "'ctx'"])
+State.ast_type('Attribute', 'expr', ['value', 'attr', 'ctx'])
 
 
 class Subscript(expr):
@@ -2514,7 +2514,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Subscript(_value, _slice, _ctx, _lineno, _col_offset)
 
-State.ast_type('Subscript', 'expr', ["'value'", "'slice'", "'ctx'"])
+State.ast_type('Subscript', 'expr', ['value', 'slice', 'ctx'])
 
 
 class Starred(expr):
@@ -2559,7 +2559,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Starred(_value, _ctx, _lineno, _col_offset)
 
-State.ast_type('Starred', 'expr', ["'value'", "'ctx'"])
+State.ast_type('Starred', 'expr', ['value', 'ctx'])
 
 
 class Name(expr):
@@ -2603,7 +2603,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Name(_id, _ctx, _lineno, _col_offset)
 
-State.ast_type('Name', 'expr', ["'id'", "'ctx'"])
+State.ast_type('Name', 'expr', ['id', 'ctx'])
 
 
 class List(expr):
@@ -2652,7 +2652,7 @@
         _col_offset = space.int_w(w_col_offset)
         return List(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type('List', 'expr', ["'elts'", "'ctx'"])
+State.ast_type('List', 'expr', ['elts', 'ctx'])
 
 
 class Tuple(expr):
@@ -2701,7 +2701,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Tuple(_elts, _ctx, _lineno, _col_offset)
 
-State.ast_type('Tuple', 'expr', ["'elts'", "'ctx'"])
+State.ast_type('Tuple', 'expr', ['elts', 'ctx'])
 
 
 class Const(expr):
@@ -2738,7 +2738,7 @@
         _col_offset = space.int_w(w_col_offset)
         return Const(_value, _lineno, _col_offset)
 
-State.ast_type('Const', 'expr', ["'value'"])
+State.ast_type('Const', 'expr', ['value'])
 
 
 class expr_context(AST):
@@ -2860,7 +2860,7 @@
         _step = expr.from_object(space, w_step) if w_step is not None else None
         return Slice(_lower, _upper, _step)
 
-State.ast_type('Slice', 'slice', ["'lower'", "'upper'", "'step'"])
+State.ast_type('Slice', 'slice', ['lower', 'upper', 'step'])
 
 
 class ExtSlice(slice):
@@ -2893,7 +2893,7 @@
         _dims = [slice.from_object(space, w_item) for w_item in dims_w]
         return ExtSlice(_dims)
 
-State.ast_type('ExtSlice', 'slice', ["'dims'"])
+State.ast_type('ExtSlice', 'slice', ['dims'])
 
 
 class Index(slice):
@@ -2922,7 +2922,7 @@
             raise_required_value(space, w_node, 'value')
         return Index(_value)
 
-State.ast_type('Index', 'slice', ["'value'"])
+State.ast_type('Index', 'slice', ['value'])
 
 
 class boolop(AST):
@@ -3276,7 +3276,7 @@
         _ifs = [expr.from_object(space, w_item) for w_item in ifs_w]
         return comprehension(_target, _iter, _ifs)
 
-State.ast_type('comprehension', 'AST', ["'target'", "'iter'", "'ifs'"])
+State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs'])
 
 class excepthandler(AST):
 
@@ -3292,7 +3292,7 @@
             return ExceptHandler.from_object(space, w_node)
         raise oefmt(space.w_TypeError,
                 "Expected excepthandler node, got %T", w_node)
-State.ast_type('excepthandler', 'AST', None, ["'lineno'", "'col_offset'"])
+State.ast_type('excepthandler', 'AST', None, ['lineno', 'col_offset'])
 
 class ExceptHandler(excepthandler):
 
@@ -3345,7 +3345,7 @@
         _col_offset = space.int_w(w_col_offset)
         return ExceptHandler(_type, _name, _body, _lineno, _col_offset)
 
-State.ast_type('ExceptHandler', 'excepthandler', ["'type'", "'name'", "'body'"])
+State.ast_type('ExceptHandler', 'excepthandler', ['type', 'name', 'body'])
 
 
 class arguments(AST):
@@ -3428,7 +3428,7 @@
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwonlyargs, _kw_defaults, _kwarg, _defaults)
 
-State.ast_type('arguments', 'AST', ["'args'", "'vararg'", "'kwonlyargs'", "'kw_defaults'", "'kwarg'", "'defaults'"])
+State.ast_type('arguments', 'AST', ['args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults'])
 
 class arg(AST):
 
@@ -3462,7 +3462,7 @@
         _annotation = expr.from_object(space, w_annotation) if w_annotation is not None else None
         return arg(_arg, _annotation)
 
-State.ast_type('arg', 'AST', ["'arg'", "'annotation'"])
+State.ast_type('arg', 'AST', ['arg', 'annotation'])
 
 class keyword(AST):
 
@@ -3495,7 +3495,7 @@
             raise_required_value(space, w_node, 'value')
         return keyword(_arg, _value)
 
-State.ast_type('keyword', 'AST', ["'arg'", "'value'"])
+State.ast_type('keyword', 'AST', ['arg', 'value'])
 
 class alias(AST):
 
@@ -3527,7 +3527,7 @@
         _asname = space.str_or_None_w(w_asname)
         return alias(_name, _asname)
 
-State.ast_type('alias', 'AST', ["'name'", "'asname'"])
+State.ast_type('alias', 'AST', ['name', 'asname'])
 
 class withitem(AST):
 
@@ -3562,7 +3562,7 @@
         _optional_vars = expr.from_object(space, w_optional_vars) if w_optional_vars is not None else None
         return withitem(_context_expr, _optional_vars)
 
-State.ast_type('withitem', 'AST', ["'context_expr'", "'optional_vars'"])
+State.ast_type('withitem', 'AST', ['context_expr', 'optional_vars'])
 
 class ASTVisitor(object):
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -102,7 +102,7 @@
             self.emit("raise oefmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
             self.emit("State.ast_type(%r, 'AST', None, %s)" %
-                      (base, [repr(attr.name) for attr in sum.attributes]))
+                      (base, [repr(attr.name)[1:-1] for attr in sum.attributes]))
             self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
@@ -119,7 +119,7 @@
         self.emit("")
         self.make_converters(product.fields, name)
         self.emit("State.ast_type(%r, 'AST', %s)" %
-                  (name, [repr(f.name) for f in product.fields]))
+                  (name, [repr(f.name)[1:-1] for f in product.fields]))
         self.emit("")
 
     def get_value_converter(self, field, value):
@@ -263,7 +263,7 @@
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
         self.emit("State.ast_type(%r, '%s', %s)" % 
-                  (cons.name, base, [repr(f.name) for f in cons.fields]))
+                  (cons.name, base, [repr(f.name)[1:-1] for f in cons.fields]))
         self.emit("")
 
     def visitField(self, field):

From pypy.commits at gmail.com  Mon Jun 13 15:22:14 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 13 Jun 2016 12:22:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Better way to fix the double quotes
Message-ID: <575f07e6.d81b1c0a.8b947.13af@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85144:caaa5d28bdd6
Date: 2016-06-13 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/caaa5d28bdd6/

Log:	Better way to fix the double quotes

diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -102,7 +102,7 @@
             self.emit("raise oefmt(space.w_TypeError,", 2)
             self.emit("        \"Expected %s node, got %%T\", w_node)" % (base,), 2)
             self.emit("State.ast_type(%r, 'AST', None, %s)" %
-                      (base, [repr(attr.name)[1:-1] for attr in sum.attributes]))
+                      (base, [attr.name for attr in sum.attributes]))
             self.emit("")
             for cons in sum.types:
                 self.visit(cons, base, sum.attributes)
@@ -119,7 +119,7 @@
         self.emit("")
         self.make_converters(product.fields, name)
         self.emit("State.ast_type(%r, 'AST', %s)" %
-                  (name, [repr(f.name)[1:-1] for f in product.fields]))
+                  (name, [f.name for f in product.fields]))
         self.emit("")
 
     def get_value_converter(self, field, value):
@@ -263,7 +263,7 @@
         self.make_mutate_over(cons, cons.name)
         self.make_converters(cons.fields, cons.name, extra_attributes)
         self.emit("State.ast_type(%r, '%s', %s)" % 
-                  (cons.name, base, [repr(f.name)[1:-1] for f in cons.fields]))
+                  (cons.name, base, [f.name for f in cons.fields]))
         self.emit("")
 
     def visitField(self, field):

From pypy.commits at gmail.com  Mon Jun 13 15:28:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Jun 2016 12:28:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Found a way in CPython to have several
 empty string objects.
Message-ID: <575f0965.49921c0a.95dae.153c@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85145:9417f63e9eff
Date: 2016-06-13 21:29 +0200
http://bitbucket.org/pypy/pypy/changeset/9417f63e9eff/

Log:	Found a way in CPython to have several empty string objects.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -335,9 +335,8 @@
 ``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
 test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
 equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
-closer to CPython's, which caches precisely the empty
-string/unicode/tuple/frozenset, and (sometimes!) the single-character
-strings and unicodes.
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,

From pypy.commits at gmail.com  Mon Jun 13 15:52:55 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 13 Jun 2016 12:52:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Return an arg (instead of identifier
 value) in handle arguments for STAR and DOUBLESTAR
Message-ID: <575f0f17.c409c20a.ad0f2.ffffe239@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85146:4ed7839a6bd6
Date: 2016-06-13 21:51 +0200
http://bitbucket.org/pypy/pypy/changeset/4ed7839a6bd6/

Log:	Return an arg (instead of identifier value) in handle arguments for
	STAR and DOUBLESTAR

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -570,9 +570,7 @@
         kwonly = [] if n_kwdonly else None
         kwdefaults = []
         kwarg = None
-        kwargann = None
         vararg = None
-        varargann = None
         if n_pos + n_kwdonly > 255:
             self.error("more than 255 arguments", arguments_node)
         # process args
@@ -604,11 +602,7 @@
                     i = self.handle_keywordonly_args(arguments_node, i, kwonly,
                                                      kwdefaults)
                 else:
-                    vararg = name_node.get_child(0).get_value()
-                    vararg = self.new_identifier(vararg)
-                    self.check_forbidden_name(vararg, name_node)
-                    if name_node.num_children() > 1:
-                        varargann = self.handle_expr(name_node.get_child(2))
+                    vararg = self.handle_arg(name_node)
                     i += 3
                     if i < child_count:
                         next_arg_type = arguments_node.get_child(i).type
@@ -618,11 +612,7 @@
                                                              kwonly, kwdefaults)
             elif arg_type == tokens.DOUBLESTAR:
                 name_node = arguments_node.get_child(i + 1)
-                kwarg = name_node.get_child(0).get_value()
-                kwarg = self.new_identifier(kwarg)
-                self.check_forbidden_name(kwarg, name_node)
-                if name_node.num_children() > 1:
-                    kwargann = self.handle_expr(name_node.get_child(2))
+                kwarg = self.handle_arg(name_node)
                 i += 3
             else:
                 raise AssertionError("unknown node in argument list")

From pypy.commits at gmail.com  Mon Jun 13 16:06:33 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 13 Jun 2016 13:06:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove vararg- and kwargannotation checks
 in visit_annotation for symtable
Message-ID: <575f1249.138f1c0a.4ebf1.229c@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85147:8cd46370914c
Date: 2016-06-13 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/8cd46370914c/

Log:	Remove vararg- and kwargannotation checks in visit_annotation for
	symtable

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -532,10 +532,6 @@
         assert isinstance(args, ast.arguments)
         if args.args:
             self._visit_arg_annotations(args.args)
-        if args.varargannotation:
-            args.varargannotation.walkabout(self)
-        if args.kwargannotation:
-            args.kwargannotation.walkabout(self)
         if args.kwonlyargs:
             self._visit_arg_annotations(args.kwonlyargs)
         if func.returns:

From pypy.commits at gmail.com  Mon Jun 13 16:35:04 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 13 Jun 2016 13:35:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix errors of identifier->arg change,
 remove more obsolete annotation checks
Message-ID: <575f18f8.cc9d1c0a.fd717.2886@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85148:7297e4bad09b
Date: 2016-06-13 22:34 +0200
http://bitbucket.org/pypy/pypy/changeset/7297e4bad09b/

Log:	Fix errors of identifier->arg change, remove more obsolete
	annotation checks

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -352,13 +352,7 @@
         space = self.space
         names = []
         self._visit_arg_annotations(args.args, names)
-        if args.varargannotation:
-            self._visit_arg_annotation(args.vararg, args.varargannotation,
-                                       names)
         self._visit_arg_annotations(args.kwonlyargs, names)
-        if args.kwargannotation:
-            self._visit_arg_annotation(args.kwarg, args.kwargannotation,
-                                       names)
         self._visit_arg_annotation("return", returns, names)
         l = len(names)
         if l:
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -516,10 +516,10 @@
         if arguments.kwonlyargs:
             self._handle_params(arguments.kwonlyargs, True)
         if arguments.vararg:
-            self.note_symbol(arguments.vararg, SYM_PARAM)
+            self.note_symbol(arguments.vararg.arg, SYM_PARAM)
             scope.note_variable_arg(arguments.vararg)
         if arguments.kwarg:
-            self.note_symbol(arguments.kwarg, SYM_PARAM)
+            self.note_symbol(arguments.kwarg.arg, SYM_PARAM)
             scope.note_keywords_arg(arguments.kwarg)
 
     def _handle_params(self, params, is_toplevel):

From pypy.commits at gmail.com  Mon Jun 13 18:36:21 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Mon, 13 Jun 2016 15:36:21 -0700 (PDT)
Subject: [pypy-commit] pypy gc-forkfriendly: Move some GC flags the JIT
 needs into where the JIT needs them (i.e. not remote_flags)
Message-ID: <575f3565.0654c20a.1573.1d98@mx.google.com>

Author: Devin Jeanpierre 
Branch: gc-forkfriendly
Changeset: r85149:bc8241225da0
Date: 2016-06-13 15:35 -0700
http://bitbucket.org/pypy/pypy/changeset/bc8241225da0/

Log:	Move some GC flags the JIT needs into where the JIT needs them (i.e.
	not remote_flags)

diff --git a/rpython/memory/gc/incminimark_remoteheader.py b/rpython/memory/gc/incminimark_remoteheader.py
--- a/rpython/memory/gc/incminimark_remoteheader.py
+++ b/rpython/memory/gc/incminimark_remoteheader.py
@@ -8,6 +8,16 @@
 
 SIGNEDP = lltype.Ptr(lltype.FixedSizeArray(lltype.Signed, 1))
 
+# HACK: because GCFLAG_CARDS_SET is checked in some ugly assembler, and the
+#       assembler is hardcoded to check the tid field. We special case that flag
+#       alone so that it can still be stored in tid!
+#       This should not really impact memory, because if a card is set, then the
+#       page was already mutated to add a young ptr, so there is no harm. It
+#       might mess with performance a little though. :)
+# TODO: but why GCFLAG_TRACK_YOUNG_PTRS?
+#       (Only figured that one out by trial/error
+NONREMOTE_FLAGS = incminimark.GCFLAG_TRACK_YOUNG_PTRS | incminimark.GCFLAG_CARDS_SET
+
 class IncrementalMiniMarkRemoteHeaderGC(incminimark.IncrementalMiniMarkGCBase):
     # The GC header is similar to incminimark, except that the flags can be
     # placed anywhere, not just in the bits of tid.
@@ -96,19 +106,29 @@
 
     def get_flags(self, obj):
         self.__lazy_init_flags(obj)
-        return self.header(obj).remote_flags[0]
+        hdr = self.header(obj)
+        return hdr.remote_flags[0] | (hdr.tid & NONREMOTE_FLAGS)
 
     def set_flags(self, obj, flags):
         self.__lazy_init_flags(obj)
-        self.header(obj).remote_flags[0] = flags
+        hdr = self.header(obj)
+        hdr.remote_flags[0] = flags & ~incminimark.GCFLAG_CARDS_SET
+        if flags & NONREMOTE_FLAGS:
+            hdr.tid = (hdr.tid & ~NONREMOTE_FLAGS ) | (flags & NONREMOTE_FLAGS )
 
     def add_flags(self, obj, flags):
         self.__lazy_init_flags(obj)
-        self.header(obj).remote_flags[0] |= flags
+        hdr = self.header(obj)
+        hdr.remote_flags[0] |= flags
+        if flags & NONREMOTE_FLAGS:
+            self.header(obj).tid |= (flags & NONREMOTE_FLAGS )
 
     def remove_flags(self, obj, flags):
         self.__lazy_init_flags(obj)
-        self.header(obj).remote_flags[0] &= ~flags
+        hdr = self.header(obj)
+        hdr.remote_flags[0] &= ~flags
+        if flags & NONREMOTE_FLAGS:
+            self.header(obj).tid &= ~(flags & NONREMOTE_FLAGS )
 
 
 def _free_flags_if_finalized(adr, unused_arg):

From pypy.commits at gmail.com  Tue Jun 14 03:48:40 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 14 Jun 2016 00:48:40 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Add include guards to pymem.h:
 fixes issue #2321
Message-ID: <575fb6d8.0410c20a.4e6ac.140e@mx.google.com>

Author: Ronan Lamy 
Branch: release-5.x
Changeset: r85150:9618737a6b92
Date: 2016-06-09 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/9618737a6b92/

Log:	Add include guards to pymem.h: fixes issue #2321 (grafted from
	68486f0f79c649514bde51317d4ab97602d11713)

diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */

From pypy.commits at gmail.com  Tue Jun 14 03:48:42 2016
From: pypy.commits at gmail.com (vext01)
Date: Tue, 14 Jun 2016 00:48:42 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Make vmprof build on OpenBSD.
Message-ID: <575fb6da.442cc20a.d70fb.ffffb2d9@mx.google.com>

Author: Edd Barrett 
Branch: release-5.x
Changeset: r85151:f4e9427ad4bf
Date: 2016-06-09 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/f4e9427ad4bf/

Log:	Make vmprof build on OpenBSD. (grafted from
	b30d44ba76f7eb2ed8ba19ba4500c62a89704f1f)

diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,14 @@
+#if !defined(__OpenBSD__)
 #define HAVE_SYS_UCONTEXT_H
+#endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,8 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(__OpenBSD__)
+#include 
 #endif
 
 

From pypy.commits at gmail.com  Tue Jun 14 03:48:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 00:48:44 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Issue #2324: fix for
 bytearray().replace('a', 'ab')
Message-ID: <575fb6dc.46c21c0a.c6ed3.ffff9b88@mx.google.com>

Author: Armin Rigo 
Branch: release-5.x
Changeset: r85152:40b8fc42f439
Date: 2016-06-13 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/40b8fc42f439/

Log:	Issue #2324: fix for bytearray().replace('a', 'ab')

	We have too much mess and code duplication around here. (grafted
	from 41928beccff95a36c2dd8731f78785b9f7872c52)

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -162,7 +162,8 @@
             buffer = _get_buffer(space, w_sub)
             res = count(value, buffer, start, end)
 
-        return space.wrap(max(res, 0))
+        assert res >= 0
+        return space.wrap(res)
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -211,6 +211,7 @@
 
         check(bytearray('abc').replace('b', bytearray('d')), 'adc')
         check(bytearray('abc').replace('b', 'd'), 'adc')
+        check(bytearray('').replace('a', 'ab'), '')
 
         check(bytearray('abc').upper(), 'ABC')
         check(bytearray('ABC').lower(), 'abc')
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -291,6 +291,7 @@
     return _search(value, other, start, end, SEARCH_COUNT)
 
 # -------------- substring searching helper ----------------
+# XXX a lot of code duplication with lltypesystem.rstr :-(
 
 SEARCH_COUNT = 0
 SEARCH_FIND = 1
@@ -309,6 +310,8 @@
     if end > len(value):
         end = len(value)
     if start > end:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     count = 0
@@ -326,6 +329,8 @@
     w = n - m
 
     if w < 0:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     mlast = m - 1
@@ -570,18 +575,20 @@
 
 class ByteListBuilder(object):
     def __init__(self, init_size=INIT_SIZE):
+        assert init_size >= 0
         self.l = newlist_hint(init_size)
 
     @specialize.argtype(1)
     def append(self, s):
+        l = self.l
         for c in s:
-            self.l.append(c)
+            l.append(c)
 
     @specialize.argtype(1)
     def append_slice(self, s, start, end):
-        assert 0 <= start <= end <= len(s)
-        for c in s[start:end]:
-            self.l.append(c)
+        l = self.l
+        for i in xrange(start, end):
+            l.append(s[i])
 
     def append_multiple_char(self, c, times):
         assert isinstance(c, str)
@@ -589,8 +596,9 @@
 
     def append_charpsize(self, s, size):
         assert size >= 0
+        l = self.l
         for i in xrange(size):
-            self.l.append(s[i])
+            l.append(s[i])
 
     def build(self):
         return self.l
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -231,6 +231,10 @@
     check_search(count, 'one two three', 'e', 0, 1, res=0)
     check_search(count, 'one two three', '', 0, 13, res=14)
 
+    check_search(count, '', 'ab', 0, 0, res=0)
+    check_search(count, 'a', 'ab', 0, 1, res=0)
+    check_search(count, 'ac', 'ab', 0, 2, res=0)
+
 
 class TestTranslates(BaseRtypingTest):
     def test_split_rsplit(self):
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -972,6 +972,13 @@
             s.count(s, -10)
         py.test.raises(AnnotatorError, self.interpret, f, ())
 
+    def test_count_in_empty_string(self):
+        const = self.const
+        def fn():
+            return const('').count(const('ab'))
+        res = self.interpret(fn, [])
+        assert res == 0
+
     def test_getitem_exc(self):
         const = self.const
         def f(x):

From pypy.commits at gmail.com  Tue Jun 14 03:48:46 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 00:48:46 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: increment rev number to 5.3.1
Message-ID: <575fb6de.08371c0a.6d664.ffffe3cc@mx.google.com>

Author: Matti Picus 
Branch: release-5.x
Changeset: r85153:7e8df3df9641
Date: 2016-06-14 10:46 +0300
http://bitbucket.org/pypy/pypy/changeset/7e8df3df9641/

Log:	increment rev number to 5.3.1

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.0"
-#define PYPY_VERSION_NUM  0x05030000
+#define PYPY_VERSION "5.3.1"
+#define PYPY_VERSION_NUM  0x05030100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 1, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue Jun 14 03:48:47 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 00:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: increment rev number to 5.3.2
Message-ID: <575fb6df.08371c0a.6d664.ffffe3d0@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85154:f43307421241
Date: 2016-06-14 10:46 +0300
http://bitbucket.org/pypy/pypy/changeset/f43307421241/

Log:	increment rev number to 5.3.2

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.1-alpha0"
-#define PYPY_VERSION_NUM  0x05030100
+#define PYPY_VERSION "5.3.2-alpha0"
+#define PYPY_VERSION_NUM  0x05030200
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue Jun 14 05:24:22 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 02:24:22 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: allow rw access to the
 pointer returned from PyString_AS_STRING(obj) if obj is newly created
Message-ID: <575fcd46.d11b1c0a.39976.0fc0@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85155:5b2e7547c432
Date: 2016-06-14 12:23 +0300
http://bitbucket.org/pypy/pypy/changeset/5b2e7547c432/

Log:	allow rw access to the pointer returned from PyString_AS_STRING(obj)
	if obj is newly created

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -6,7 +6,8 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref)
+    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
+    pyobj_has_w_obj)
 
 ##
 ## Implementation of PyStringObject
@@ -41,6 +42,9 @@
 ##   the pypy string is created, and added to the global map of tracked
 ##   objects.  The buffer is then supposed to be immutable.
 ##
+##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
+##   there is no corresponding pypy object for the string
+##
 ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
 ##   similar object.
 ##
@@ -139,6 +143,9 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
+    return _PyString_AsString(space, ref)
+
+def _PyString_AsString(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         pass    # typecheck returned "ok" without forcing 'ref' at all
     elif not PyString_Check(space, ref):   # otherwise, use the alternate way
@@ -158,6 +165,19 @@
         ref_str.c_buffer = rffi.str2charp(s)
     return ref_str.c_buffer
 
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
+def PyString_AS_STRING(space, void_ref):
+    ref = rffi.cast(PyObject, void_ref)
+    # if no w_str is associated with this ref,
+    # return the c-level ptr as RW
+    if not pyobj_has_w_obj(ref):
+        py_str = rffi.cast(PyStringObject, ref)
+        if not py_str.c_buffer:
+            py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
+                                        flavor='raw', zero=True)
+        return py_str.c_buffer
+    return _PyString_AsString(space, ref)
+
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
 def PyString_AsStringAndSize(space, ref, buffer, length):
     if not PyString_Check(space, ref):
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -10,7 +10,6 @@
 #include 
 
 #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op))
-#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op))
 /*
 Type PyStringObject represents a character string.  An extra zero byte is
 reserved at the end to ensure it is zero-terminated, but a size is
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -110,14 +110,23 @@
                 obj = (PyStringObject*)type->tp_alloc(type, 10);
                 if (PyString_GET_SIZE(obj) != 10)
                     return PyLong_FromLong(PyString_GET_SIZE(obj));
-                /* cannot work, there is only RO access
-                memcpy(PyString_AS_STRING(obj), "works", 6); */
+                /* cannot work, there is only RO access */
+                memcpy(PyString_AS_STRING(obj), "works", 6);
                 Py_INCREF(obj);
                 return (PyObject*)obj;
              """),
+            ('alloc_rw', "METH_NOARGS",
+             '''
+                PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
+                char * buf = PyString_AS_STRING(obj);
+                memcpy(PyString_AS_STRING(obj), "works", 6);
+                return (PyObject*)obj;
+             '''),
             ])
+        s = module.alloc_rw()
+        assert s == 'works' + '\x00' * 5
         s = module.tpalloc()
-        assert s == '\x00' * 10
+        assert s == 'works' + '\x00' * 5
 
     def test_AsString(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Tue Jun 14 05:42:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 14 Jun 2016 02:42:38 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: copy paste vector_ext file to
 provide a skeleton to ppc, 
Message-ID: <575fd18e.56311c0a.7c69a.173b@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85156:2cef040e9515
Date: 2016-06-13 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/2cef040e9515/

Log:	copy paste vector_ext file to provide a skeleton to ppc, update vec
	argument to be turned on by default now updated doc of the option

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -0,0 +1,725 @@
+import py
+from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.history import (ConstInt, INT, REF,
+    FLOAT, VECTOR, TargetToken)
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.llsupport.regalloc import get_scale
+from rpython.jit.metainterp.resoperation import (rop, ResOperation,
+        VectorOp, VectorGuardOp)
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
+
+def not_implemented(msg):
+    msg = '[ppc/vector_ext] %s\n' % msg
+    if we_are_translated():
+        llop.debug_print(lltype.Void, msg)
+    raise NotImplementedError(msg)
+
+class PPCVectorAssemblerMixin(object):
+    _mixin_ = True
+
+    #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
+    #    self.implement_guard(guard_token)
+
+    #def genop_guard_vec_guard_false(self, guard_op, guard_token, locs, resloc):
+    #    self.guard_success_cc = rx86.invert_condition(self.guard_success_cc)
+    #    self.implement_guard(guard_token)
+
+    #def guard_vector(self, guard_op, loc, true):
+    #    assert isinstance(guard_op, VectorGuardOp)
+    #    arg = guard_op.getarg(0)
+    #    assert isinstance(arg, VectorOp)
+    #    size = arg.bytesize
+    #    temp = X86_64_XMM_SCRATCH_REG
+    #    load = arg.bytesize * arg.count - self.cpu.vector_register_size
+    #    assert load <= 0
+    #    if true:
+    #        self.mc.PXOR(temp, temp)
+    #        # if the vector is not fully packed blend 1s
+    #        if load < 0:
+    #            self.mc.PCMPEQQ(temp, temp) # fill with ones
+    #            self._blend_unused_slots(loc, arg, temp)
+    #            # reset to zeros
+    #            self.mc.PXOR(temp, temp)
+
+    #        # cmp with zeros (in temp) creates ones at each slot where it is zero
+    #        self.mc.PCMPEQ(loc, temp, size)
+    #        # temp converted to ones
+    #        self.mc.PCMPEQQ(temp, temp)
+    #        # test if all slots are zero
+    #        self.mc.PTEST(loc, temp)
+    #        self.guard_success_cc = rx86.Conditions['Z']
+    #    else:
+    #        # if the vector is not fully packed blend 1s
+    #        if load < 0:
+    #            temp = X86_64_XMM_SCRATCH_REG
+    #            self.mc.PXOR(temp, temp)
+    #            self._blend_unused_slots(loc, arg, temp)
+    #        self.mc.PTEST(loc, loc)
+    #        self.guard_success_cc = rx86.Conditions['NZ']
+
+    #def _blend_unused_slots(self, loc, arg, temp):
+    #    select = 0
+    #    bits_used = (arg.count * arg.bytesize * 8)
+    #    index = bits_used // 16
+    #    while index < 8:
+    #        select |= (1 << index)
+    #        index += 1
+    #    self.mc.PBLENDW_xxi(loc.value, temp.value, select)
+
+    #def _update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
+    #    """ If accumulation is done in this loop, at the guard exit
+    #        some vector registers must be adjusted to yield the correct value
+    #    """
+    #    if not isinstance(faildescr, ResumeGuardDescr):
+    #        return
+    #    assert regalloc is not None
+    #    accum_info = faildescr.rd_vector_info
+    #    while accum_info:
+    #        pos = accum_info.getpos_in_failargs()
+    #        scalar_loc = fail_locs[pos]
+    #        vector_loc = accum_info.location
+    #        # the upper elements will be lost if saved to the stack!
+    #        scalar_arg = accum_info.getoriginal()
+    #        assert isinstance(vector_loc, RegLoc)
+    #        if not isinstance(scalar_loc, RegLoc):
+    #            scalar_loc = regalloc.force_allocate_reg(scalar_arg)
+    #        assert scalar_arg is not None
+    #        if accum_info.accum_operation == '+':
+    #            self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc)
+    #        elif accum_info.accum_operation == '*':
+    #            self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc)
+    #        else:
+    #            not_implemented("accum operator %s not implemented" %
+    #                                        (accum_info.accum_operation)) 
+    #        accum_info = accum_info.next()
+
+    #def _accum_reduce_mul(self, arg, accumloc, targetloc):
+    #    scratchloc = X86_64_XMM_SCRATCH_REG
+    #    self.mov(accumloc, scratchloc)
+    #    # swap the two elements
+    #    self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01)
+    #    self.mc.MULSD(accumloc, scratchloc)
+    #    if accumloc is not targetloc:
+    #        self.mov(accumloc, targetloc)
+
+    #def _accum_reduce_sum(self, arg, accumloc, targetloc):
+    #    # Currently the accumulator can ONLY be the biggest
+    #    # size for X86 -> 64 bit float/int
+    #    if arg.type == FLOAT:
+    #        # r = (r[0]+r[1],r[0]+r[1])
+    #        self.mc.HADDPD(accumloc, accumloc)
+    #        # upper bits (> 64) are dirty (but does not matter)
+    #        if accumloc is not targetloc:
+    #            self.mov(accumloc, targetloc)
+    #        return
+    #    elif arg.type == INT:
+    #        scratchloc = X86_64_SCRATCH_REG
+    #        self.mc.PEXTRQ_rxi(targetloc.value, accumloc.value, 0)
+    #        self.mc.PEXTRQ_rxi(scratchloc.value, accumloc.value, 1)
+    #        self.mc.ADD(targetloc, scratchloc)
+    #        return
+
+    #    not_implemented("reduce sum for %s not impl." % arg)
+
+    #def _genop_vec_getarrayitem(self, op, arglocs, resloc):
+    #    # considers item scale (raw_load does not)
+    #    base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+    #    scale = get_scale(size_loc.value)
+    #    src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
+    #    self._vec_load(resloc, src_addr, integer_loc.value,
+    #                   size_loc.value, aligned_loc.value)
+    #
+    #genop_vec_getarrayitem_raw_i = _genop_vec_getarrayitem
+    #genop_vec_getarrayitem_raw_f = _genop_vec_getarrayitem
+    #
+    #genop_vec_getarrayitem_gc_i = _genop_vec_getarrayitem
+    #genop_vec_getarrayitem_gc_f = _genop_vec_getarrayitem
+
+    #def _genop_vec_raw_load(self, op, arglocs, resloc):
+    #    base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+    #    src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
+    #    self._vec_load(resloc, src_addr, integer_loc.value,
+    #                   size_loc.value, aligned_loc.value)
+
+    #genop_vec_raw_load_i = _genop_vec_raw_load
+    #genop_vec_raw_load_f = _genop_vec_raw_load
+
+    #def _vec_load(self, resloc, src_addr, integer, itemsize, aligned):
+    #    if integer:
+    #        if aligned:
+    #            self.mc.MOVDQA(resloc, src_addr)
+    #        else:
+    #            self.mc.MOVDQU(resloc, src_addr)
+    #    else:
+    #        if itemsize == 4:
+    #            self.mc.MOVUPS(resloc, src_addr)
+    #        elif itemsize == 8:
+    #            self.mc.MOVUPD(resloc, src_addr)
+
+    #def _genop_discard_vec_setarrayitem(self, op, arglocs):
+    #    # considers item scale (raw_store does not)
+    #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
+    #    scale = get_scale(size_loc.value)
+    #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
+    #    self._vec_store(dest_loc, value_loc, integer_loc.value,
+    #                    size_loc.value, aligned_loc.value)
+
+    #genop_discard_vec_setarrayitem_raw = _genop_discard_vec_setarrayitem
+    #genop_discard_vec_setarrayitem_gc = _genop_discard_vec_setarrayitem
+
+    #def genop_discard_vec_raw_store(self, op, arglocs):
+    #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
+    #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
+    #    self._vec_store(dest_loc, value_loc, integer_loc.value,
+    #                    size_loc.value, aligned_loc.value)
+
+    #def _vec_store(self, dest_loc, value_loc, integer, itemsize, aligned):
+    #    if integer:
+    #        if aligned:
+    #            self.mc.MOVDQA(dest_loc, value_loc)
+    #        else:
+    #            self.mc.MOVDQU(dest_loc, value_loc)
+    #    else:
+    #        if itemsize == 4:
+    #            self.mc.MOVUPS(dest_loc, value_loc)
+    #        elif itemsize == 8:
+    #            self.mc.MOVUPD(dest_loc, value_loc)
+
+    #def genop_vec_int_is_true(self, op, arglocs, resloc):
+    #    loc, sizeloc = arglocs
+    #    temp = X86_64_XMM_SCRATCH_REG
+    #    self.mc.PXOR(temp, temp)
+    #    # every entry that is non zero -> becomes zero
+    #    # zero entries become ones
+    #    self.mc.PCMPEQ(loc, temp, sizeloc.value)
+    #    # a second time -> every zero entry (corresponding to non zero
+    #    # entries before) become ones
+    #    self.mc.PCMPEQ(loc, temp, sizeloc.value)
+
+    #def genop_vec_int_mul(self, op, arglocs, resloc):
+    #    loc0, loc1, itemsize_loc = arglocs
+    #    itemsize = itemsize_loc.value
+    #    if itemsize == 2:
+    #        self.mc.PMULLW(loc0, loc1)
+    #    elif itemsize == 4:
+    #        self.mc.PMULLD(loc0, loc1)
+    #    else:
+    #        # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
+    #        # There is no 64x64 bit packed mul. For 8 bit either. It is questionable if it gives any benefit?
+    #        not_implemented("int8/64 mul")
+
+    #def genop_vec_int_add(self, op, arglocs, resloc):
+    #    loc0, loc1, size_loc = arglocs
+    #    size = size_loc.value
+    #    if size == 1:
+    #        self.mc.PADDB(loc0, loc1)
+    #    elif size == 2:
+    #        self.mc.PADDW(loc0, loc1)
+    #    elif size == 4:
+    #        self.mc.PADDD(loc0, loc1)
+    #    elif size == 8:
+    #        self.mc.PADDQ(loc0, loc1)
+
+    #def genop_vec_int_sub(self, op, arglocs, resloc):
+    #    loc0, loc1, size_loc = arglocs
+    #    size = size_loc.value
+    #    if size == 1:
+    #        self.mc.PSUBB(loc0, loc1)
+    #    elif size == 2:
+    #        self.mc.PSUBW(loc0, loc1)
+    #    elif size == 4:
+    #        self.mc.PSUBD(loc0, loc1)
+    #    elif size == 8:
+    #        self.mc.PSUBQ(loc0, loc1)
+
+    #def genop_vec_int_and(self, op, arglocs, resloc):
+    #    self.mc.PAND(resloc, arglocs[0])
+
+    #def genop_vec_int_or(self, op, arglocs, resloc):
+    #    self.mc.POR(resloc, arglocs[0])
+
+    #def genop_vec_int_xor(self, op, arglocs, resloc):
+    #    self.mc.PXOR(resloc, arglocs[0])
+
+    #genop_vec_float_arith = """
+    #def genop_vec_float_{type}(self, op, arglocs, resloc):
+    #    loc0, loc1, itemsize_loc = arglocs
+    #    itemsize = itemsize_loc.value
+    #    if itemsize == 4:
+    #        self.mc.{p_op_s}(loc0, loc1)
+    #    elif itemsize == 8:
+    #        self.mc.{p_op_d}(loc0, loc1)
+    #"""
+    #for op in ['add','mul','sub']:
+    #    OP = op.upper()
+    #    _source = genop_vec_float_arith.format(type=op,
+    #                                           p_op_s=OP+'PS',
+    #                                           p_op_d=OP+'PD')
+    #    exec py.code.Source(_source).compile()
+    #del genop_vec_float_arith
+
+    #def genop_vec_float_truediv(self, op, arglocs, resloc):
+    #    loc0, loc1, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    if size == 4:
+    #        self.mc.DIVPS(loc0, loc1)
+    #    elif size == 8:
+    #        self.mc.DIVPD(loc0, loc1)
+
+    #def genop_vec_float_abs(self, op, arglocs, resloc):
+    #    src, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    if size == 4:
+    #        self.mc.ANDPS(src, heap(self.single_float_const_abs_addr))
+    #    elif size == 8:
+    #        self.mc.ANDPD(src, heap(self.float_const_abs_addr))
+
+    #def genop_vec_float_neg(self, op, arglocs, resloc):
+    #    src, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    if size == 4:
+    #        self.mc.XORPS(src, heap(self.single_float_const_neg_addr))
+    #    elif size == 8:
+    #        self.mc.XORPD(src, heap(self.float_const_neg_addr))
+
+    #def genop_vec_float_eq(self, op, arglocs, resloc):
+    #    _, rhsloc, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    if size == 4:
+    #        self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 0) # 0 means equal
+    #    else:
+    #        self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 0)
+
+    #def genop_vec_float_ne(self, op, arglocs, resloc):
+    #    _, rhsloc, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    # b(100) == 1 << 2 means not equal
+    #    if size == 4:
+    #        self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 1 << 2)
+    #    else:
+    #        self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 1 << 2)
+
+    #def genop_vec_int_eq(self, op, arglocs, resloc):
+    #    _, rhsloc, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    self.mc.PCMPEQ(resloc, rhsloc, size)
+
+    #def genop_vec_int_ne(self, op, arglocs, resloc):
+    #    _, rhsloc, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    self.mc.PCMPEQ(resloc, rhsloc, size)
+    #    temp = X86_64_XMM_SCRATCH_REG
+    #    self.mc.PCMPEQQ(temp, temp) # set all bits to one
+    #    # need to invert the value in resloc
+    #    self.mc.PXOR(resloc, temp)
+    #    # 11 00 11 11
+    #    # 11 11 11 11
+    #    # ----------- pxor
+    #    # 00 11 00 00
+
+    #def genop_vec_int_signext(self, op, arglocs, resloc):
+    #    srcloc, sizeloc, tosizeloc = arglocs
+    #    size = sizeloc.value
+    #    tosize = tosizeloc.value
+    #    if size == tosize:
+    #        return # already the right size
+    #    if size == 4 and tosize == 8:
+    #        scratch = X86_64_SCRATCH_REG.value
+    #        self.mc.PEXTRD_rxi(scratch, srcloc.value, 1)
+    #        self.mc.PINSRQ_xri(resloc.value, scratch, 1)
+    #        self.mc.PEXTRD_rxi(scratch, srcloc.value, 0)
+    #        self.mc.PINSRQ_xri(resloc.value, scratch, 0)
+    #    elif size == 8 and tosize == 4:
+    #        # is there a better sequence to move them?
+    #        scratch = X86_64_SCRATCH_REG.value
+    #        self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0)
+    #        self.mc.PINSRD_xri(resloc.value, scratch, 0)
+    #        self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1)
+    #        self.mc.PINSRD_xri(resloc.value, scratch, 1)
+    #    else:
+    #        # note that all other conversions are not implemented
+    #        # on purpose. it needs many x86 op codes to implement
+    #        # the missing combinations. even if they are implemented
+    #        # the speedup might only be modest...
+    #        # the optimization does not emit such code!
+    #        msg = "vec int signext (%d->%d)" % (size, tosize)
+    #        not_implemented(msg)
+
+    #def genop_vec_expand_f(self, op, arglocs, resloc):
+    #    srcloc, sizeloc = arglocs
+    #    size = sizeloc.value
+    #    if isinstance(srcloc, ConstFloatLoc):
+    #        # they are aligned!
+    #        self.mc.MOVAPD(resloc, srcloc)
+    #    elif size == 4:
+    #        # the register allocator forces src to be the same as resloc
+    #        # r = (s[0], s[0], r[0], r[0])
+    #        # since resloc == srcloc: r = (r[0], r[0], r[0], r[0])
+    #        self.mc.SHUFPS_xxi(resloc.value, srcloc.value, 0)
+    #    elif size == 8:
+    #        self.mc.MOVDDUP(resloc, srcloc)
+    #    else:
+    #        raise AssertionError("float of size %d not supported" % (size,))
+
+    #def genop_vec_expand_i(self, op, arglocs, resloc):
+    #    srcloc, sizeloc = arglocs
+    #    if not isinstance(srcloc, RegLoc):
+    #        self.mov(srcloc, X86_64_SCRATCH_REG)
+    #        srcloc = X86_64_SCRATCH_REG
+    #    assert not srcloc.is_xmm
+    #    size = sizeloc.value
+    #    if size == 1:
+    #        self.mc.PINSRB_xri(resloc.value, srcloc.value, 0)
+    #        self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr))
+    #    elif size == 2:
+    #        self.mc.PINSRW_xri(resloc.value, srcloc.value, 0)
+    #        self.mc.PINSRW_xri(resloc.value, srcloc.value, 4)
+    #        self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0)
+    #        self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0)
+    #    elif size == 4:
+    #        self.mc.PINSRD_xri(resloc.value, srcloc.value, 0)
+    #        self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0)
+    #    elif size == 8:
+    #        self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0)
+    #        self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1)
+    #    else:
+    #        raise AssertionError("cannot handle size %d (int expand)" % (size,))
+
+    #def genop_vec_pack_i(self, op, arglocs, resloc):
+    #    resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
+    #    assert isinstance(resultloc, RegLoc)
+    #    assert isinstance(sourceloc, RegLoc)
+    #    size = sizeloc.value
+    #    srcidx = srcidxloc.value
+    #    residx = residxloc.value
+    #    count = countloc.value
+    #    # for small data type conversion this can be quite costy
+    #    # NOTE there might be some combinations that can be handled
+    #    # more efficiently! e.g.
+    #    # v2 = pack(v0,v1,4,4)
+    #    si = srcidx
+    #    ri = residx
+    #    k = count
+    #    while k > 0:
+    #        if size == 8:
+    #            if resultloc.is_xmm and sourceloc.is_xmm: # both xmm
+    #                self.mc.PEXTRQ_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+    #                self.mc.PINSRQ_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+    #            elif resultloc.is_xmm: # xmm <- reg
+    #                self.mc.PINSRQ_xri(resultloc.value, sourceloc.value, ri)
+    #            else: # reg <- xmm
+    #                self.mc.PEXTRQ_rxi(resultloc.value, sourceloc.value, si)
+    #        elif size == 4:
+    #            if resultloc.is_xmm and sourceloc.is_xmm:
+    #                self.mc.PEXTRD_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+    #                self.mc.PINSRD_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+    #            elif resultloc.is_xmm:
+    #                self.mc.PINSRD_xri(resultloc.value, sourceloc.value, ri)
+    #            else:
+    #                self.mc.PEXTRD_rxi(resultloc.value, sourceloc.value, si)
+    #        elif size == 2:
+    #            if resultloc.is_xmm and sourceloc.is_xmm:
+    #                self.mc.PEXTRW_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+    #                self.mc.PINSRW_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+    #            elif resultloc.is_xmm:
+    #                self.mc.PINSRW_xri(resultloc.value, sourceloc.value, ri)
+    #            else:
+    #                self.mc.PEXTRW_rxi(resultloc.value, sourceloc.value, si)
+    #        elif size == 1:
+    #            if resultloc.is_xmm and sourceloc.is_xmm:
+    #                self.mc.PEXTRB_rxi(X86_64_SCRATCH_REG.value, sourceloc.value, si)
+    #                self.mc.PINSRB_xri(resultloc.value, X86_64_SCRATCH_REG.value, ri)
+    #            elif resultloc.is_xmm:
+    #                self.mc.PINSRB_xri(resultloc.value, sourceloc.value, ri)
+    #            else:
+    #                self.mc.PEXTRB_rxi(resultloc.value, sourceloc.value, si)
+    #        si += 1
+    #        ri += 1
+    #        k -= 1
+
+    #genop_vec_unpack_i = genop_vec_pack_i
+
+    #def genop_vec_pack_f(self, op, arglocs, resultloc):
+    #    resloc, srcloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
+    #    assert isinstance(resloc, RegLoc)
+    #    assert isinstance(srcloc, RegLoc)
+    #    count = countloc.value
+    #    residx = residxloc.value
+    #    srcidx = srcidxloc.value
+    #    size = sizeloc.value
+    #    if size == 4:
+    #        si = srcidx
+    #        ri = residx
+    #        k = count
+    #        while k > 0:
+    #            if resloc.is_xmm:
+    #                src = srcloc.value
+    #                if not srcloc.is_xmm:
+    #                    # if source is a normal register (unpack)
+    #                    assert count == 1
+    #                    assert si == 0
+    #                    self.mov(srcloc, X86_64_XMM_SCRATCH_REG)
+    #                    src = X86_64_XMM_SCRATCH_REG.value
+    #                select = ((si & 0x3) << 6)|((ri & 0x3) << 4)
+    #                self.mc.INSERTPS_xxi(resloc.value, src, select)
+    #            else:
+    #                self.mc.PEXTRD_rxi(resloc.value, srcloc.value, si)
+    #            si += 1
+    #            ri += 1
+    #            k -= 1
+    #    elif size == 8:
+    #        assert resloc.is_xmm
+    #        if srcloc.is_xmm:
+    #            if srcidx == 0:
+    #                if residx == 0:
+    #                    # r = (s[0], r[1])
+    #                    self.mc.MOVSD(resloc, srcloc)
+    #                else:
+    #                    assert residx == 1
+    #                    # r = (r[0], s[0])
+    #                    self.mc.UNPCKLPD(resloc, srcloc)
+    #            else:
+    #                assert srcidx == 1
+    #                if residx == 0:
+    #                    # r = (s[1], r[1])
+    #                    if resloc != srcloc:
+    #                        self.mc.UNPCKHPD(resloc, srcloc)
+    #                    self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+    #                else:
+    #                    assert residx == 1
+    #                    # r = (r[0], s[1])
+    #                    if resloc != srcloc:
+    #                        self.mc.SHUFPD_xxi(resloc.value, resloc.value, 1)
+    #                        self.mc.UNPCKHPD(resloc, srcloc)
+    #                    # if they are equal nothing is to be done
+
+    #genop_vec_unpack_f = genop_vec_pack_f
+
+    #def genop_vec_cast_float_to_singlefloat(self, op, arglocs, resloc):
+    #    self.mc.CVTPD2PS(resloc, arglocs[0])
+
+    #def genop_vec_cast_float_to_int(self, op, arglocs, resloc):
+    #    self.mc.CVTPD2DQ(resloc, arglocs[0])
+
+    #def genop_vec_cast_int_to_float(self, op, arglocs, resloc):
+    #    self.mc.CVTDQ2PD(resloc, arglocs[0])
+
+    #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, resloc):
+    #    self.mc.CVTPS2PD(resloc, arglocs[0])
+
+clas#s VectorRegallocMixin(object):
+    #_mixin_ = True
+
+    #def _consider_vec_getarrayitem(self, op):
+    #    descr = op.getdescr()
+    #    assert isinstance(descr, ArrayDescr)
+    #    assert not descr.is_array_of_pointers() and \
+    #           not descr.is_array_of_structs()
+    #    itemsize, ofs, _ = unpack_arraydescr(descr)
+    #    integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+    #    aligned = False
+    #    args = op.getarglist()
+    #    base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+    #    ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+    #    result_loc = self.force_allocate_reg(op)
+    #    self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
+    #                      imm(integer), imm(aligned)], result_loc)
+
+    #consider_vec_getarrayitem_raw_i = _consider_vec_getarrayitem
+    #consider_vec_getarrayitem_raw_f = _consider_vec_getarrayitem
+    #consider_vec_getarrayitem_gc_i = _consider_vec_getarrayitem
+    #consider_vec_getarrayitem_gc_f = _consider_vec_getarrayitem
+    #consider_vec_raw_load_i = _consider_vec_getarrayitem
+    #consider_vec_raw_load_f = _consider_vec_getarrayitem
+
+    #def _consider_vec_setarrayitem(self, op):
+    #    descr = op.getdescr()
+    #    assert isinstance(descr, ArrayDescr)
+    #    assert not descr.is_array_of_pointers() and \
+    #           not descr.is_array_of_structs()
+    #    itemsize, ofs, _ = unpack_arraydescr(descr)
+    #    args = op.getarglist()
+    #    base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+    #    value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+    #    ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+
+    #    integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+    #    aligned = False
+    #    self.perform_discard(op, [base_loc, ofs_loc, value_loc,
+    #                             imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
+
+    #consider_vec_setarrayitem_raw = _consider_vec_setarrayitem
+    #consider_vec_setarrayitem_gc = _consider_vec_setarrayitem
+    #consider_vec_raw_store = _consider_vec_setarrayitem
+
+    #def consider_vec_arith(self, op):
+    #    lhs = op.getarg(0)
+    #    assert isinstance(op, VectorOp)
+    #    size = op.bytesize
+    #    args = op.getarglist()
+    #    loc1 = self.make_sure_var_in_reg(op.getarg(1), args)
+    #    loc0 = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    self.perform(op, [loc0, loc1, imm(size)], loc0)
+
+    #consider_vec_int_add = consider_vec_arith
+    #consider_vec_int_sub = consider_vec_arith
+    #consider_vec_int_mul = consider_vec_arith
+    #consider_vec_float_add = consider_vec_arith
+    #consider_vec_float_sub = consider_vec_arith
+    #consider_vec_float_mul = consider_vec_arith
+    #consider_vec_float_truediv = consider_vec_arith
+    #del consider_vec_arith
+
+    #def consider_vec_arith_unary(self, op):
+    #    lhs = op.getarg(0)
+    #    assert isinstance(lhs, VectorOp)
+    #    args = op.getarglist()
+    #    res = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    self.perform(op, [res, imm(lhs.bytesize)], res)
+
+    #consider_vec_float_neg = consider_vec_arith_unary
+    #consider_vec_float_abs = consider_vec_arith_unary
+    #del consider_vec_arith_unary
+
+    #def consider_vec_logic(self, op):
+    #    lhs = op.getarg(0)
+    #    assert isinstance(lhs, VectorOp)
+    #    args = op.getarglist()
+    #    source = self.make_sure_var_in_reg(op.getarg(1), args)
+    #    result = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    self.perform(op, [source, imm(lhs.bytesize)], result)
+
+    #def consider_vec_float_eq(self, op):
+    #    assert isinstance(op, VectorOp)
+    #    lhs = op.getarg(0)
+    #    assert isinstance(lhs, VectorOp)
+    #    args = op.getarglist()
+    #    rhsloc = self.make_sure_var_in_reg(op.getarg(1), args)
+    #    lhsloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    self.perform(op, [lhsloc, rhsloc, imm(lhs.bytesize)], lhsloc)
+
+    #consider_vec_float_ne = consider_vec_float_eq
+    #consider_vec_int_eq = consider_vec_float_eq
+    #consider_vec_int_ne = consider_vec_float_eq
+
+    #consider_vec_int_and = consider_vec_logic
+    #consider_vec_int_or = consider_vec_logic
+    #consider_vec_int_xor = consider_vec_logic
+    #del consider_vec_logic
+
+    #def consider_vec_pack_i(self, op):
+    #    # new_res = vec_pack_i(res, src, index, count)
+    #    assert isinstance(op, VectorOp)
+    #    arg = op.getarg(1)
+    #    index = op.getarg(2)
+    #    count = op.getarg(3)
+    #    assert isinstance(index, ConstInt)
+    #    assert isinstance(count, ConstInt)
+    #    args = op.getarglist()
+    #    srcloc = self.make_sure_var_in_reg(arg, args)
+    #    resloc =  self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    residx = index.value # where to put it in result?
+    #    srcidx = 0
+    #    arglocs = [resloc, srcloc, imm(residx), imm(srcidx),
+    #               imm(count.value), imm(op.bytesize)]
+    #    self.perform(op, arglocs, resloc)
+
+    #consider_vec_pack_f = consider_vec_pack_i
+
+    #def consider_vec_unpack_i(self, op):
+    #    assert isinstance(op, VectorOp)
+    #    index = op.getarg(1)
+    #    count = op.getarg(2)
+    #    assert isinstance(index, ConstInt)
+    #    assert isinstance(count, ConstInt)
+    #    args = op.getarglist()
+    #    srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
+    #    if op.is_vector():
+    #        resloc =  self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #        size = op.bytesize
+    #    else:
+    #        # unpack into iX box
+    #        resloc =  self.force_allocate_reg(op, args)
+    #        arg = op.getarg(0)
+    #        assert isinstance(arg, VectorOp)
+    #        size = arg.bytesize
+    #    residx = 0
+    #    args = op.getarglist()
+    #    arglocs = [resloc, srcloc, imm(residx), imm(index.value), imm(count.value), imm(size)]
+    #    self.perform(op, arglocs, resloc)
+
+    #consider_vec_unpack_f = consider_vec_unpack_i
+
+    #def consider_vec_expand_f(self, op):
+    #    assert isinstance(op, VectorOp)
+    #    arg = op.getarg(0)
+    #    args = op.getarglist()
+    #    if arg.is_constant():
+    #        resloc = self.xrm.force_allocate_reg(op)
+    #        srcloc = self.xrm.expand_float(op.bytesize, arg)
+    #    else:
+    #        resloc = self.xrm.force_result_in_reg(op, arg, args)
+    #        srcloc = resloc
+    #    self.perform(op, [srcloc, imm(op.bytesize)], resloc)
+
+    #def consider_vec_expand_i(self, op):
+    #    assert isinstance(op, VectorOp)
+    #    arg = op.getarg(0)
+    #    args = op.getarglist()
+    #    if arg.is_constant():
+    #        srcloc = self.rm.convert_to_imm(arg)
+    #    else:
+    #        srcloc = self.make_sure_var_in_reg(arg, args)
+    #    resloc = self.xrm.force_allocate_reg(op, args)
+    #    self.perform(op, [srcloc, imm(op.bytesize)], resloc)
+
+    #def consider_vec_int_signext(self, op):
+    #    assert isinstance(op, VectorOp)
+    #    args = op.getarglist()
+    #    resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    arg = op.getarg(0)
+    #    assert isinstance(arg, VectorOp)
+    #    size = arg.bytesize
+    #    assert size > 0
+    #    self.perform(op, [resloc, imm(size), imm(op.bytesize)], resloc)
+
+    #def consider_vec_int_is_true(self, op):
+    #    args = op.getarglist()
+    #    arg = op.getarg(0)
+    #    assert isinstance(arg, VectorOp)
+    #    argloc = self.loc(arg)
+    #    resloc = self.xrm.force_result_in_reg(op, arg, args)
+    #    self.perform(op, [resloc,imm(arg.bytesize)], None)
+
+    #def _consider_vec(self, op):
+    #    # pseudo instruction, needed to create a new variable
+    #    self.xrm.force_allocate_reg(op)
+
+    #consider_vec_i = _consider_vec
+    #consider_vec_f = _consider_vec
+
+    #def consider_vec_cast_float_to_int(self, op):
+    #    args = op.getarglist()
+    #    srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
+    #    resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
+    #    self.perform(op, [srcloc], resloc)
+
+    #consider_vec_cast_int_to_float = consider_vec_cast_float_to_int
+    #consider_vec_cast_float_to_singlefloat = consider_vec_cast_float_to_int
+    #consider_vec_cast_singlefloat_to_float = consider_vec_cast_float_to_int
+
+    #def consider_vec_guard_true(self, op):
+    #    arg = op.getarg(0)
+    #    loc = self.loc(arg)
+    #    self.assembler.guard_vector(op, self.loc(arg), True)
+    #    self.perform_guard(op, [], None)
+
+    #def consider_vec_guard_false(self, op):
+    #    arg = op.getarg(0)
+    #    loc = self.loc(arg)
+    #    self.assembler.guard_vector(op, self.loc(arg), False)
+    #    self.perform_guard(op, [], None)
+
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -552,7 +552,8 @@
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
-    'vec': 'turn on the vectorization optimization (vecopt). requires sse4.1',
+    'vec': 'turn on the vectorization optimization (vecopt). ' \
+           'Supports powerpc (SVX), x86 (SSE 4.1)',
     'vec_all': 'try to vectorize trace loops that occur outside of the numpy library.',
     'vec_cost': 'threshold for which traces to bail. 0 means the costs.',
     'vec_length': 'the amount of instructions allowed in "all" traces.',
@@ -575,7 +576,7 @@
               'disable_unrolling': 200,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
-              'vec': 0,
+              'vec': 1,
               'vec_all': 0,
               'vec_cost': 0,
               'vec_length': 60,

From pypy.commits at gmail.com  Tue Jun 14 05:42:40 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 14 Jun 2016 02:42:40 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: first test testing unaligned
 load
Message-ID: <575fd190.8b371c0a.f17e.13c9@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85157:33b15420f68d
Date: 2016-06-14 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/33b15420f68d/

Log:	first test testing unaligned load

diff --git a/rpython/jit/backend/ppc/test/test_vector_instr.py b/rpython/jit/backend/ppc/test/test_vector_instr.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/test/test_vector_instr.py
@@ -0,0 +1,67 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+from rpython.jit.backend.ppc.codebuilder import BasicPPCAssembler, PPCBuilder
+from rpython.jit.backend.ppc.regname import *
+from rpython.jit.backend.ppc.register import *
+from rpython.jit.backend.ppc import form
+from rpython.jit.backend import detect_cpu
+from rpython.jit.backend.ppc.arch import IS_PPC_32, IS_PPC_64, IS_BIG_ENDIAN
+from rpython.jit.backend.ppc.arch import WORD
+
+cpu = detect_cpu.autodetect()
+
+signed = lltype.Signed
+unsigned = lltype.Unsigned
+char = lltype.Char
+
+def vec_asmtest(memory=[]):
+    def testmaker(test):
+        def newtest(self):
+            memory_ptrs = []
+            a = PPCBuilder()
+            for (bytes, type, values) in memory:
+                # alloc
+                adr = lltype.malloc(rffi.CArray(char), bytes, flavor="raw")
+                memory_ptrs.append(adr)
+                address = adr
+                for i,value in enumerate(values):
+                    rffi.cast(rffi.CArrayPtr(type), adr)[i] = rffi.cast(type, value)
+
+            expected = test(self, a, *[rffi.cast(lltype.Signed, m) for m in memory_ptrs])
+            f = a.get_assembler_function()
+            f()
+            for type, expect, ptr in expected:
+                value = rffi.cast(lltype.CArrayPtr(type), ptr)[0]
+                assert value == expect
+
+            while memory_ptrs:
+                ptr = memory_ptrs.pop()
+                lltype.free(ptr, flavor="raw")
+        return newtest
+    return testmaker
+
+
+class TestAssemble(object):
+    """
+    See how tests are built in test_ppc.py
+
+    Instead of asmtest, use vec_asmtest. It adds a parameter 'memory' that
+    allocates memory.
+
+
+    @vec_asmtest(memory=[(8, [1,2]), 'm2': (16, [1,2,3,4])])
+    def test(self, builder, m, m2):
+        # ...
+        return [ (value, type, ptr), ... ]
+    """
+    def setup_class(cls):
+        if cpu not in ["ppc", "ppc64", "ppc-64"]:
+            py.test.skip("can't test all of ppcgen on non-PPC!")
+
+    @vec_asmtest(memory=[(8, signed, [0,0])])
+    def test_unaligned_load(self, a, mem):
+        a.load_imm(r15, mem)
+        a.lxvd2x(0, 15, mem)
+        a.blr()
+        return [ (0, signed, mem), (0, signed, mem+8) ]
+

From pypy.commits at gmail.com  Tue Jun 14 08:32:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 14 Jun 2016 05:32:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: more changes and impl. of
 lxvd2x test passes now
Message-ID: <575ff972.a91cc20a.d7d7b.3aac@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85158:d326e675ae9d
Date: 2016-06-14 14:31 +0200
http://bitbucket.org/pypy/pypy/changeset/d326e675ae9d/

Log:	more changes and impl. of lxvd2x test passes now

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -1,4 +1,5 @@
 import os
+
 from rpython.jit.backend.ppc.ppc_form import PPCForm as Form
 from rpython.jit.backend.ppc.locations import RegisterLocation
 from rpython.jit.backend.ppc.ppc_field import ppc_fields
@@ -60,6 +61,7 @@
 XFL = Form("FM", "frB", "XO1", "Rc")
 XFX = Form("CRM", "rS", "XO1")
 XLL = Form("LL", "XO1")
+XX1 = Form("vrT", "rA", "rB", "XO1")
 
 MI = Form("rA", "rS", "SH", "MB", "ME", "Rc")
 MB = Form("rA", "rS", "rB", "MB", "ME", "Rc")
@@ -266,6 +268,7 @@
     lwbrx = XD(31, XO1=534)
     lwzux = XD(31, XO1=55)
     lwzx  = XD(31, XO1=23)
+    lxvd2x = XX1(31, XO1=844)
 
     mcrfs  = Form("crfD", "crfS", "XO1")(63, XO1=64)
     mcrxr  = Form("crfD", "XO1")(31, XO1=512)
diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py
--- a/rpython/jit/backend/ppc/locations.py
+++ b/rpython/jit/backend/ppc/locations.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.history import INT, FLOAT
+from rpython.jit.metainterp.history import INT, FLOAT, VECTOR
 import sys
 
 # cannot import from arch.py, currently we have a circular import
@@ -75,6 +75,24 @@
     def as_key(self):
         return self.value + 100
 
+class VectorRegisterLocation(AssemblerLocation):
+    _immutable_ = True
+    width = WORD
+    type = VECTOR
+
+    def __init__(self, value):
+        self.value = value
+
+    def __repr__(self):
+        return 'vr%d' % self.value
+
+    def is_reg(self):
+        return True
+
+    def as_key(self):
+        return self.value + 132
+
+
 class ImmLocation(AssemblerLocation):
     _immutable_ = True
     width = WORD
diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py
--- a/rpython/jit/backend/ppc/ppc_field.py
+++ b/rpython/jit/backend/ppc/ppc_field.py
@@ -43,6 +43,7 @@
     "spr":    (11, 20),
     "TO":     ( 6, 10),
     "UIMM":   (16, 31),
+    "vrT":    (6,  31, 'unsigned', regname._V, 'overlap'),
     "XO1":    (21, 30),
     "XO2":    (22, 30),
     "XO3":    (26, 30),
@@ -100,6 +101,17 @@
     def decode(self, inst):
         value = super(sh, self).decode(inst)
         return (value & 32) << 5 | (value >> 10 & 31)
+
+class tx(Field):
+    def encode(self, value):
+        value = (value & 31) << 20 | (value & 32) >> 5
+        return super(tx, self).encode(value)
+    def decode(self, inst):
+        value = super(tx, self).decode(inst)
+        return (value & 32) << 5 | (value >> 20 & 31)
+    def r(self):
+        import pdb; pdb.set_trace()
+        return super(tx, self).r()
 # other special fields?
 
 ppc_fields = {
@@ -109,6 +121,7 @@
     "mbe": mbe("mbe",   *fields["mbe"]),
     "sh":  sh("sh",     *fields["sh"]),
     "spr": spr("spr",   *fields["spr"]),
+    "vrT": tx("vrT",    *fields["vrT"]),
 }
 
 for f in fields:
diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py
--- a/rpython/jit/backend/ppc/rassemblermaker.py
+++ b/rpython/jit/backend/ppc/rassemblermaker.py
@@ -46,6 +46,9 @@
         elif field.name == 'sh':
             body.append('sh1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value))
             value = 'sh1'
+        elif field.name == 'vrT':
+            body.append('vrT1 = (%s & 31) << 20 | (%s & 32) >> 5' % (value, value))
+            value = 'vrT1'
         if isinstance(field, IField):
             body.append('v |= ((%3s >> 2) & r_uint(%#05x)) << 2' % (value, field.mask))
         else:
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -1,8 +1,9 @@
 from rpython.jit.backend.ppc.locations import (RegisterLocation,
-                                               FPRegisterLocation)
+       FPRegisterLocation, VectorRegisterLocation)
 
 ALL_REGS        = [RegisterLocation(i) for i in range(32)]
 ALL_FLOAT_REGS  = [FPRegisterLocation(i) for i in range(32)]
+ALL_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(64)]
 
 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16,\
     r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31\
@@ -12,6 +13,13 @@
     f17, f18, f19, f20, f21, f22, f23, f24, f25, f26, f27, f28, f29, f30, f31\
     = ALL_FLOAT_REGS
 
+vr0, vr1, vr2, vr3, vr4, vr5, vr6, vr7, vr8, vr9, vr10, vr11, vr12, vr13, \
+     vr14, vr15, vr16, vr17, vr18, vr19, vr20, vr21, vr22, vr23, vr24, vr25, \
+     vr26, vr27, vr28, vr29, vr30, vr31, vr32, vr33, vr34, vr35, vr36, vr37, \
+     vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \
+     vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \
+     vr61, vr62, vr63 = ALL_VECTOR_REGS
+
 NONVOLATILES        = [r14, r15, r16, r17, r18, r19, r20, r21, r22, r23,
                     r24, r25, r26, r27, r28, r29, r30, r31]
 VOLATILES           = [r3, r4, r5, r6, r7, r8, r9, r10, r11, r12]
diff --git a/rpython/jit/backend/ppc/regname.py b/rpython/jit/backend/ppc/regname.py
--- a/rpython/jit/backend/ppc/regname.py
+++ b/rpython/jit/backend/ppc/regname.py
@@ -6,6 +6,10 @@
     def __repr__(self):
         return "fr%s"%(super(_F, self).__repr__(),)
     __str__ = __repr__
+class _V(int):
+    def __repr__(self):
+        return "vr%s"%(super(_V, self).__repr__(),)
+    __str__ = __repr__
 
 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, \
     r13, r14, r15, r16, r17, r18, r19, r20, r21, r22, \
@@ -15,4 +19,11 @@
      fr13, fr14, fr15, fr16, fr17, fr18, fr19, fr20, fr21, fr22, \
      fr23, fr24, fr25, fr26, fr27, fr28, fr29, fr30, fr31 = map(_F, range(32))
 
+vr0, vr1, vr2, vr3, vr4, vr5, vr6, vr7, vr8, vr9, vr10, vr11, vr12, vr13, \
+     vr14, vr15, vr16, vr17, vr18, vr19, vr20, vr21, vr22, vr23, vr24, vr25, \
+     vr26, vr27, vr28, vr29, vr30, vr31, vr32, vr33, vr34, vr35, vr36, vr37, \
+     vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \
+     vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \
+     vr61, vr62, vr63 = map(_V, range(64))
+
 crf0, crf1, crf2, crf3, crf4, crf5, crf6, crf7 = range(8)
diff --git a/rpython/jit/backend/ppc/test/test_vector_instr.py b/rpython/jit/backend/ppc/test/test_vector_instr.py
--- a/rpython/jit/backend/ppc/test/test_vector_instr.py
+++ b/rpython/jit/backend/ppc/test/test_vector_instr.py
@@ -30,8 +30,8 @@
             expected = test(self, a, *[rffi.cast(lltype.Signed, m) for m in memory_ptrs])
             f = a.get_assembler_function()
             f()
-            for type, expect, ptr in expected:
-                value = rffi.cast(lltype.CArrayPtr(type), ptr)[0]
+            for expect, type, ptr in expected:
+                value = rffi.cast(rffi.CArrayPtr(type), ptr)[0]
                 assert value == expect
 
             while memory_ptrs:
@@ -61,7 +61,7 @@
     @vec_asmtest(memory=[(8, signed, [0,0])])
     def test_unaligned_load(self, a, mem):
         a.load_imm(r15, mem)
-        a.lxvd2x(0, 15, mem)
+        a.lxvd2x(vr0.value, 0, r15.value)
         a.blr()
         return [ (0, signed, mem), (0, signed, mem+8) ]
 

From pypy.commits at gmail.com  Tue Jun 14 08:48:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 14 Jun 2016 05:48:46 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added vector store instruction
 and test for a load store combination
Message-ID: <575ffd2e.08371c0a.6d664.69ae@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85159:9072f071d260
Date: 2016-06-14 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/9072f071d260/

Log:	added vector store instruction and test for a load store combination

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -268,7 +268,6 @@
     lwbrx = XD(31, XO1=534)
     lwzux = XD(31, XO1=55)
     lwzx  = XD(31, XO1=23)
-    lxvd2x = XX1(31, XO1=844)
 
     mcrfs  = Form("crfD", "crfS", "XO1")(63, XO1=64)
     mcrxr  = Form("crfD", "XO1")(31, XO1=512)
@@ -571,7 +570,21 @@
     xor = XS(31, XO1=316, Rc=0)
     xorx = XS(31, XO1=316, Rc=1)
 
-class PPCAssembler(BasicPPCAssembler):
+    #
+
+class PPCVSXAssembler(object):
+    _mixin_ = True
+
+    # load
+    lxvdsx = XX1(31, XO1=332) # splat first element
+    lxvd2x = XX1(31, XO1=844)
+    lxvw4x = XX1(31, XO1=780)
+
+    # store
+    stxvd2x = XX1(31, XO1=972)
+    stxvw4x = XX1(31, XO1=908)
+
+class PPCAssembler(BasicPPCAssembler, PPCVSXAssembler):
     BA = BasicPPCAssembler
 
     # awkward mnemonics:
diff --git a/rpython/jit/backend/ppc/test/test_vector_instr.py b/rpython/jit/backend/ppc/test/test_vector_instr.py
--- a/rpython/jit/backend/ppc/test/test_vector_instr.py
+++ b/rpython/jit/backend/ppc/test/test_vector_instr.py
@@ -58,10 +58,19 @@
         if cpu not in ["ppc", "ppc64", "ppc-64"]:
             py.test.skip("can't test all of ppcgen on non-PPC!")
 
-    @vec_asmtest(memory=[(8, signed, [0,0])])
+    @vec_asmtest(memory=[(16, signed, [0,0])])
     def test_unaligned_load(self, a, mem):
         a.load_imm(r15, mem)
         a.lxvd2x(vr0.value, 0, r15.value)
         a.blr()
         return [ (0, signed, mem), (0, signed, mem+8) ]
 
+    @vec_asmtest(memory=[(16, signed, [1,2]), (16, signed, [0,0])])
+    def test_unaligned_load_and_store(self, a, mem_l, mem_t):
+        a.load_imm(r15, mem_l)
+        a.load_imm(r14, mem_t)
+        a.lxvd2x(vr0.value, 0, r15.value)
+        a.stxvd2x(vr0.value, 0, r14.value)
+        a.blr()
+        return [ (1, signed, mem_t), (2, signed, mem_t+8) ]
+

From pypy.commits at gmail.com  Tue Jun 14 10:34:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 07:34:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: document the 5.3.1 release
Message-ID: <576015f5.872b1c0a.c9c5f.ffff97bc@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85160:ca2e5af3d16d
Date: 2016-06-14 17:33 +0300
http://bitbucket.org/pypy/pypy/changeset/ca2e5af3d16d/

Log:	document the 5.3.1 release

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix bytearray().replace('a', 'ab'), issue #2324

From pypy.commits at gmail.com  Tue Jun 14 10:37:52 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 07:37:52 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: add a failing test for
 subclassing PyString_Type
Message-ID: <576016c0.2457c20a.ec0a.647f@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85161:99d360889bce
Date: 2016-06-14 17:36 +0300
http://bitbucket.org/pypy/pypy/changeset/99d360889bce/

Log:	add a failing test for subclassing PyString_Type

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -341,6 +341,107 @@
         # doesn't really test, but if printf is enabled will prove sstate
         assert module.test_sstate()
 
+    def test_subclass(self):
+        # taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        module = self.import_extension('bar', [
+            ("newsubstr", "METH_O",
+             """
+                PyObject * obj;
+                char * data;
+                int len;
+                PyType_Ready(&PyStringArrType_Type);
+                
+                data = PyString_AS_STRING(args);
+                len = PyString_GET_SIZE(args);
+                if (data == NULL || len < 1)
+                    Py_RETURN_NONE;
+                obj = PyArray_Scalar(data, len);
+                return obj;
+             """),
+            ], prologue="""
+                #include 
+                PyTypeObject PyStringArrType_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "bar.string_",                /* tp_name*/
+                    sizeof(PyStringObject), /* tp_basicsize*/
+                    0                             /* tp_itemsize */
+                    };
+
+                    static PyObject *
+                    stringtype_repr(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    static PyObject *
+                    stringtype_str(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    PyObject *
+                    PyArray_Scalar(char *data, int n)
+                    {
+                        PyTypeObject *type = &PyStringArrType_Type;
+                        PyObject *obj;
+                        void *destptr;
+                        int type_num;
+                        int itemsize = n;
+                        obj = type->tp_alloc(type, itemsize);
+                        if (obj == NULL) {
+                            return NULL;
+                        }
+                        destptr = PyString_AS_STRING(obj);
+                        ((PyStringObject *)obj)->ob_shash = -1;
+                        memcpy(destptr, data, itemsize);
+                        return obj;
+                    }
+            """, more_init = '''
+                PyStringArrType_Type.tp_alloc = NULL;
+                PyStringArrType_Type.tp_free = NULL;
+
+                PyStringArrType_Type.tp_repr = stringtype_repr;
+                PyStringArrType_Type.tp_str = stringtype_str;
+                PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
+                PyStringArrType_Type.tp_itemsize = sizeof(char);
+                PyStringArrType_Type.tp_base = &PyString_Type;
+            ''')
+
+        a = module.newsubstr('abc')
+        typ = str(type(a))
+        assert  'bar.string_' in typ
+        assert a == 'abc'
 
 class TestString(BaseApiTest):
     def test_string_resize(self, space, api):

From pypy.commits at gmail.com  Tue Jun 14 11:53:59 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 14 Jun 2016 08:53:59 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix a bug in the
 guard_compatible handling in optimizeopt.
Message-ID: <57602897.0654c20a.1573.ffff80ca@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85162:349d440cf935
Date: 2016-06-14 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/349d440cf935/

Log:	fix a bug in the guard_compatible handling in optimizeopt.

	(if the arguments to the call_pure were non-constants that the
	optimizer would discover to be constants, things would break)

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -56,6 +56,13 @@
 
     def prepare_const_arg_call(self, op, optimizer):
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
+        # replace further arguments by constants, if the optimizer knows them
+        # already
+        for i in range(2, op.numargs()):
+            arg = op.getarg(i)
+            constarg = optimizer.get_constant_box(arg)
+            if constarg is not None:
+                op.setarg(i, constarg)
         copied_op = op.copy()
         copied_op.setarg(1, self.known_valid)
         if op.numargs() == 2:
@@ -147,11 +154,17 @@
         return ""
 
 class PureCallCondition(Condition):
+    const_args_start_at = 2
+
     def __init__(self, op, optimizer):
+        from rpython.jit.metainterp.history import Const
         Condition.__init__(self, optimizer)
         args = op.getarglist()[:]
         args[1] = None
         self.args = args
+        for index in range(self.const_args_start_at, len(args)):
+            arg = args[index]
+            assert isinstance(arg, Const)
         self.descr = op.getdescr()
         self.rpyfunc = op.rpyfunc
 
@@ -210,14 +223,11 @@
 
 
 class QuasiimmutGetfieldAndPureCallCondition(PureCallCondition):
+    const_args_start_at = 3
+
     def __init__(self, op, qmutdescr, optimizer):
-        Condition.__init__(self, optimizer)
-        args = op.getarglist()[:]
-        args[1] = None
-        args[2] = None
-        self.args = args
-        self.descr = op.getdescr()
-        self.rpyfunc = op.rpyfunc
+        PureCallCondition.__init__(self, op, optimizer)
+        self.args[2] = None
         self.qmut = qmutdescr.qmut
         self.mutatefielddescr = qmutdescr.mutatefielddescr
         self.fielddescr = qmutdescr.fielddescr
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
@@ -35,14 +35,14 @@
     def test_guard_compatible_and_guard_nonnull(self):
         ops = """
         [p1]
-        guard_nonnull(p1, ConstClass(node_vtable)) []
+        guard_nonnull(p1) []
         guard_compatible(p1, ConstPtr(myptr)) []
-        guard_nonnull(p1, ConstClass(node_vtable)) []
+        guard_nonnull(p1) []
         jump(ConstPtr(myptr))
         """
         expected = """
         [p1]
-        guard_nonnull(p1, ConstClass(node_vtable)) []
+        guard_nonnull(p1) []
         guard_compatible(p1, ConstPtr(myptr)) []
         jump(ConstPtr(myptr))
         """
@@ -126,6 +126,37 @@
             assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.myptr))
             assert len(descr._compatibility_conditions.conditions) == 2
 
+    def test_guard_compatible_call_pure_late_constant(self):
+        call_pure_results = {
+            (ConstInt(123), ConstPtr(self.myptr), ConstInt(5)): ConstInt(5),
+            (ConstInt(124), ConstPtr(self.myptr), ConstInt(5)): ConstInt(7),
+        }
+        ops = """
+        [p1]
+        pvirtual = new_with_vtable(descr=nodesize)
+        setfield_gc(pvirtual, 5, descr=valuedescr)
+        i1 = getfield_gc_i(pvirtual, descr=valuedescr)
+        guard_compatible(p1, ConstPtr(myptr)) []
+        i3 = call_pure_i(123, p1, i1, descr=plaincalldescr)
+        escape_n(i3)
+        i5 = call_pure_i(124, p1, i1, descr=plaincalldescr)
+        escape_n(i5)
+        jump(ConstPtr(myptr))
+        """
+        expected = """
+        [p1]
+        guard_compatible(p1, ConstPtr(myptr)) []
+        escape_n(5)
+        escape_n(7)
+        jump(ConstPtr(myptr))
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+        # whitebox-test the guard_compatible descr a bit
+        descr = self.loop.operations[1].getdescr()
+        assert descr._compatibility_conditions is not None
+        assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.myptr))
+        assert len(descr._compatibility_conditions.conditions) == 2
+
     def test_deduplicate_conditions(self):
         call_pure_results = {
             (ConstInt(123), ConstPtr(self.myptr)): ConstInt(5),
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -384,7 +384,7 @@
     so its consquences are reflected into jitted code """
     # during testing we return something randomly, to emulate the real
     # behaviour where you can switch to tracing a arbitrary points.
-    return random.random() > 0.5
+    return False
 
 _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */',
                                      default=0)

From pypy.commits at gmail.com  Tue Jun 14 12:09:20 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 09:09:20 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: allow subclassing
 PyString_Type
Message-ID: <57602c30.832c1c0a.146ba.ffffbb41@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85163:45be8ea8df59
Date: 2016-06-14 19:04 +0300
http://bitbucket.org/pypy/pypy/changeset/45be8ea8df59/

Log:	allow subclassing PyString_Type

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -8,6 +8,7 @@
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
     pyobj_has_w_obj)
+from pypy.objspace.std.bytesobject import W_BytesObject
 
 ##
 ## Implementation of PyStringObject
@@ -110,7 +111,9 @@
         py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
                                     flavor='raw', zero=True)
     s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
-    w_obj = space.wrap(s)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(W_BytesObject, w_type)
+    w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -439,8 +439,7 @@
             ''')
 
         a = module.newsubstr('abc')
-        typ = str(type(a))
-        assert  'bar.string_' in typ
+        assert type(a).__name__ == 'string_'
         assert a == 'abc'
 
 class TestString(BaseApiTest):

From pypy.commits at gmail.com  Tue Jun 14 12:23:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 14 Jun 2016 09:23:51 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Pass the C files directly to
 platform.compile() instead of hiding them in the eci; rename some parameters
Message-ID: <57602f97.0148c20a.49abc.5ebd@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85164:01b4e3b36180
Date: 2016-06-14 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/01b4e3b36180/

Log:	Pass the C files directly to platform.compile() instead of hiding
	them in the eci; rename some parameters

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -41,25 +41,22 @@
     return files
 
 def create_so(modname, include_dirs,
-        separate_module_sources=None,
-        separate_module_files=None,
+        source_strings=None,
+        source_files=None,
         **kwds):
     dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    if separate_module_sources:
-        assert not separate_module_files
-        files = convert_sources_to_files(separate_module_sources, dirname)
-        separate_module_files = files
-    eci = ExternalCompilationInfo(
-        include_dirs=include_dirs,
-        separate_module_files=separate_module_files,
-        **kwds)
-    soname = platform.platform.compile(
-        [], eci,
+    if source_strings:
+        assert not source_files
+        files = convert_sources_to_files(source_strings, dirname)
+        source_files = files
+    eci = ExternalCompilationInfo(include_dirs=include_dirs, **kwds)
+    soname = platform.platform.compile(source_files, eci,
         outputfilename=str(dirname/modname),
         standalone=False)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[], **kwds):
+def compile_extension_module(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -70,6 +67,7 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
+    kwds = {}
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
@@ -91,13 +89,16 @@
     modname = modname.split('.')[-1]
     soname = create_so(modname,
             include_dirs=api.include_dirs + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
             **kwds)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
     return str(pydname)
 
-def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds):
+def compile_extension_module_applevel(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -108,6 +109,7 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
+    kwds = {}
     if sys.platform == 'win32':
         kwds["compile_extra"] = ["/we4013"]
         kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -119,6 +121,8 @@
     modname = modname.split('.')[-1]
     soname = create_so(modname,
             include_dirs=[space.include_dir] + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
             **kwds)
     return str(soname)
 
@@ -284,8 +288,8 @@
                 separate_module_sources = []
             pydname = self.compile_extension_module(
                 space, name,
-                separate_module_files=separate_module_files,
-                separate_module_sources=separate_module_sources)
+                source_files=separate_module_files,
+                source_strings=separate_module_sources)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -330,16 +334,16 @@
                 """ % dict(name=name, init=init, body=body,
                            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
                                             if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(separate_module_sources=[code])
+                kwds = dict(source_strings=[code])
             else:
                 assert not PY_SSIZE_T_CLEAN
                 if filename is None:
                     filename = name
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(separate_module_files=[filename])
-            kwds['include_dirs'] = include_dirs
-            mod = self.compile_extension_module(space, name, **kwds)
+                kwds = dict(source_files=[filename])
+            mod = self.compile_extension_module(space, name,
+                    include_dirs=include_dirs, **kwds)
 
             if load_it:
                 if self.runappdirect:

From pypy.commits at gmail.com  Tue Jun 14 12:56:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 09:56:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: hint that the problem is only about
 empty bytearray().
Message-ID: <57603726.d12f1c0a.c2286.ffffd97f@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85165:f85dcbc231d8
Date: 2016-06-14 18:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f85dcbc231d8/

Log:	hint that the problem is only about empty bytearray().

diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.1.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -12,4 +12,4 @@
 
   * Make vmprof build on OpenBSD, from pull request #456
 
-  * Fix bytearray().replace('a', 'ab'), issue #2324
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324

From pypy.commits at gmail.com  Tue Jun 14 14:06:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 11:06:30 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Change the API of
 get_unique_id(), start implementing it
Message-ID: <576047a6.d12f1c0a.c2286.fffff237@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85166:b0f1d132948c
Date: 2016-06-14 20:07 +0200
http://bitbucket.org/pypy/pypy/changeset/b0f1d132948c/

Log:	Change the API of get_unique_id(), start implementing it

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -29,7 +29,7 @@
 
         fields = [("hash", lltype.Signed)]
         if translator.config.translation.reverse_debugger:
-            fields.append(("ctime", lltype.SignedLongLong))
+            fields.append(("uid", lltype.SignedLongLong))
         self.HDR = lltype.Struct("header", *fields)
         HDRPTR = lltype.Ptr(self.HDR)
 
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -1,12 +1,11 @@
 import sys
 from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import lltype, rstr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.annlowlevel import llhelper, hlstr
-from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
-from rpython.rtyper.rclass import OBJECTPTR
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 
 
 def stop_point():
@@ -45,24 +44,6 @@
     as the total number of stop-points)."""
     return llop.revdb_get_value(lltype.SignedLongLong, 't')
 
- at specialize.argtype(0)
-def creation_time_of(x):
-    """Returns the time at which the object 'x' was created.
-    More precisely, returns t such that object 'x' was created when
-    current_time()==t; this means that the object exists at the stop
-    point number t+1, but does not exist yet at the stop point number t.
-    """
-    return llop.revdb_creation_time_of(lltype.SignedLongLong, x)
-
- at specialize.argtype(0)
-def object_to_id(x):
-    return llop.revdb_object_to_id(lltype.Signed, x)
-
- at specialize.arg(0)
-def id_to_object(Class, object_id):
-    x = llop.revdb_id_to_object(OBJECTPTR, object_id)
-    return cast_base_ptr_to_instance(Class, x)
-
 @specialize.arg(1)
 def go_forward(time_delta, callback, arg_string):
     """For RPython debug commands: tells that after this function finishes,
@@ -81,6 +62,46 @@
     """
     _change_time('g', target_time, callback, arg_string)
 
+def currently_created_objects():
+    """For RPython debug commands: returns the current value of
+    the object creation counter.  All objects created so far have
+    a lower unique id; all objects created afterwards will have a
+    unique id greater or equal."""
+    return llop.revdb_get_value(lltype.SignedLongLong, 'u')
+
+ at specialize.argtype(0)
+def get_unique_id(x):
+    """Returns the creation number of the object 'x'.  For objects created
+    by the program, it is globally unique, monotonic, and reproducible
+    among multiple processes.  For objects created by a debug command,
+    this returns a (random) negative number.  Right now, this returns 0
+    for all prebuilt objects.
+    """
+    return llop.revdb_get_unique_id(lltype.SignedLongLong, x)
+
+def track_objects(unique_id):
+    """Track the creation of the object given by its unique_id, which must
+    be in the future (i.e. >= currently_created_objects()).  Call this
+    before go_forward().  If go_forward() goes over the creation of this
+    object, then afterwards, get_tracked_object() returns the object.
+    Going forward is also interrupted at the following stop point.
+    """
+    return llop.revdb_track_object(lltype.Bool, x)
+
+ at specialize.arg(0)
+def get_tracked_object(Class=llmemory.GCREF):   # or an RPython class
+    """Get the tracked object if it was created during the last go_forward().
+    Otherwise, returns None.  (Note: this API is minimal: to get an
+    object from its unique id, you need first to search backward for a
+    time where currently_created_objects() is lower than the unique_id,
+    then use track_object() and go_forward() to come back.  You can't
+    really track several objects, only one.)
+    """
+    x = llop.revdb_get_tracked_object(llmemory.GCREF)
+    if Class is llmemory.GCREF:
+        return x
+    return cast_gcref_to_instance(Class, x)
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -571,9 +571,9 @@
     'revdb_get_value':      LLOp(sideeffects=False),
     'revdb_set_value':      LLOp(),
     'revdb_identityhash':   LLOp(),
-    'revdb_creation_time_of': LLOp(sideeffects=False),
-    'revdb_object_to_id':   LLOp(sideeffects=False),
-    'revdb_id_to_object':   LLOp(sideeffects=False),
+    'revdb_get_unique_id':  LLOp(sideeffects=False),
+    'revdb_track_object':   LLOp(),
+    'revdb_get_tracked_object': LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -586,7 +586,7 @@
             is_atomic)
         if self.db.reverse_debugger:
             from rpython.translator.revdb import revdb_genc
-            res += revdb_genc.record_malloc_ctime(expr_result)
+            res += revdb_genc.record_malloc_uid(expr_result)
         return res
 
     def OP_BOEHM_MALLOC(self, op):
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -18,8 +18,8 @@
         return emit_void(normal_code)
     return 'RPY_REVDB_EMIT(%s, %s, %s);' % (normal_code, cdecl(tp, '_e'), value)
 
-def record_malloc_ctime(expr):
-    return ' RPY_REVDB_REC_CTIME(%s);' % (expr,)
+def record_malloc_uid(expr):
+    return ' RPY_REVDB_REC_UID(%s);' % (expr,)
 
 
 def prepare_database(db):
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -104,6 +104,7 @@
     assert(RPY_RDB_REPLAY == 0);
     rpy_revdb.buf_p = rpy_rev_buffer;
     rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32;
+    rpy_revdb.unique_id_seen = 1;
 
     if (filename && *filename) {
         putenv("PYPYRDB=");
@@ -225,6 +226,7 @@
 static uint64_t most_recent_fork;
 static uint64_t total_stop_points;
 static uint64_t stopped_time;
+static uint64_t stopped_uid;
 
 static void (*invoke_after_forward)(RPyString *);
 static RPyString *invoke_argument;
@@ -324,6 +326,7 @@
     rpy_revdb.buf_p = rpy_rev_buffer;
     rpy_revdb.buf_limit = rpy_rev_buffer;
     rpy_revdb.stop_point_break = 1;
+    rpy_revdb.unique_id_seen = 1;
 
     if (pipe(frozen_pipe_signal) < 0) {
         perror("pipe");
@@ -781,6 +784,8 @@
     };
     while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
         stopped_time = rpy_revdb.stop_point_seen;
+        stopped_uid = rpy_revdb.unique_id_seen;
+        rpy_revdb.unique_id_seen = (-1ULL) << 63;
         if (invoke_after_forward != NULL) {
             execute_rpy_function(invoke_after_forward, invoke_argument);
         }
@@ -795,7 +800,9 @@
             }
             process_input(input, "command", 1, actions_1);
         }
+        rpy_revdb.unique_id_seen = stopped_uid;
         stopped_time = 0;
+        stopped_uid = 0;
     }
 }
 
@@ -856,12 +863,21 @@
         return most_recent_fork;
     case 't':       /* total_time() */
         return total_stop_points;
-    case 'b':
+    case 'b':       /* current_break_time() */
         return rpy_revdb.stop_point_break;
+    case 'u':       /* currently_created_objects() */
+        return stopped_uid ? stopped_uid : rpy_revdb.unique_id_seen;
     default:
         return -1;
     }
 }
 
+RPY_EXTERN
+uint64_t rpy_reverse_db_unique_id_break(void *new_object)
+{
+    fprintf(stderr, "PING\n");
+    abort();
+}
+
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -18,6 +18,7 @@
 #endif
     char *buf_p, *buf_limit;
     uint64_t stop_point_seen, stop_point_break;
+    uint64_t unique_id_seen, unique_id_break;
 } rpy_revdb_t;
 
 RPY_EXTERN rpy_revdb_t rpy_revdb;
@@ -67,10 +68,14 @@
 #define RPY_REVDB_EMIT_VOID(normal_code)                                \
     if (!RPY_RDB_REPLAY) { normal_code } else { }
 
-#define RPY_REVDB_REC_CTIME(expr)                                       \
-    if (expr)                                                           \
-        ((struct pypy_header0 *)expr)->h_ctime = rpy_revdb.stop_point_seen
-
+#define RPY_REVDB_REC_UID(expr)                                         \
+    do {                                                                \
+        uint64_t uid = rpy_revdb.unique_id_seen;                        \
+        if (uid == rpy_revdb.unique_id_break || !expr)                  \
+            uid = rpy_reverse_db_unique_id_break(expr);                 \
+        rpy_revdb.unique_id_seen = uid + 1;                             \
+        ((struct pypy_header0 *)expr)->h_uid = uid;                     \
+    } while (0)
 
 #define OP_REVDB_STOP_POINT(r)                                          \
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
@@ -88,14 +93,8 @@
 #define OP_REVDB_IDENTITYHASH(obj, r)                                   \
     r = rpy_reverse_db_identityhash((struct pypy_header0 *)(obj))
 
-#define OP_REVDB_CREATION_TIME_OF(x, r)                                 \
-    r = ((struct pypy_header0 *)x)->h_ctime
-
-#define OP_REVDB_OBJECT_TO_ID(x, r)                                     \
-    r = (Signed)x
-
-#define OP_REVDB_ID_TO_OBJECT(x, r)                                     \
-    r = (void *)x
+#define OP_REVDB_GET_UNIQUE_ID(x, r)                                    \
+    r = ((struct pypy_header0 *)x)->h_uid
 
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
@@ -107,6 +106,7 @@
                                            void callback(RPyString *),
                                            RPyString *arg);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
+RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -337,11 +337,13 @@
             if cmdline == 'set-break-after-0':
                 dbstate.break_after = 0
             if cmdline == 'print-id':
-                revdb.send_output('%d\n' % (revdb.object_to_id(dbstate.stuff),))
-            if cmdline.startswith('check-id '):
-                obj_id = int(cmdline[len('check-id '):])
-                revdb.send_output("%d\n" %
-                    int(revdb.id_to_object(Stuff, obj_id) is dbstate.stuff))
+                revdb.send_output('%d %d\n' % (
+                    revdb.get_unique_id(dbstate.stuff),
+                    revdb.currently_created_objects()))
+            #if cmdline.startswith('check-id '):
+            #    obj_id = int(cmdline[len('check-id '):])
+            #    revdb.send_output("%d\n" %
+            #        int(revdb.id_to_object(Stuff, obj_id) is dbstate.stuff))
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -350,9 +352,10 @@
         dbstate = DBState()
         #
         def main(argv):
-            dbstate.stuff = Stuff()
             revdb.register_debug_command('r', lambda_blip)
             for i, op in enumerate(argv[1:]):
+                dbstate.stuff = Stuff()
+                dbstate.stuff.x = i + 1000
                 revdb.stop_point()
                 if i == dbstate.break_after:
                     revdb.send_output('breakpoint!\n')
@@ -440,6 +443,7 @@
                       '(3)$ ')
 
     def test_dynamic_breakpoint(self):
+        py.test.skip("unsure if that's needed")
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('__go 1')
@@ -450,16 +454,19 @@
         child.expectx('breakpoint!\r\n'
                       '(2)$ ')
 
-    def test_object_to_id(self):
+    def test_get_unique_id(self):
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r print-id')
         child.expect(re.escape('<<>>\r\n')
-                     + r'(-?\d+)'
+                     + r'(\d+) (\d+)'
                      + re.escape('\r\n'
                                  'blipped\r\n'
                                  '(3)$ '))
-        object_id = child.match.group(1)
+        object_id = int(child.match.group(1))
+        currenty_created_objects = int(child.match.group(2))
+        assert 0 < object_id < currenty_created_objects
+        XXX
         for at_time in [1, 2, 3]:
             child.sendline('__go %d' % at_time)
             child.expectx('(%d)$ ' % at_time)

From pypy.commits at gmail.com  Tue Jun 14 14:07:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 14 Jun 2016 11:07:26 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Use DistutilsPlatform as a
 stepping stone to removing the dependency on rpython.platform
Message-ID: <576047de.cbc71c0a.2fe96.ffffeab9@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85167:670eb478dec3
Date: 2016-06-14 19:06 +0100
http://bitbucket.org/pypy/pypy/changeset/670eb478dec3/

Log:	Use DistutilsPlatform as a stepping stone to removing the dependency
	on rpython.platform

	As a side-effect, this enables -Werror for C modules defined in
	test_*.py files, so fix some of the warnings.

diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -31,7 +31,7 @@
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
                      printf("tp_basicsize==%ld\\n",
-                            (long)s->ob_type->tp_basicsize); 
+                            (long)s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -53,7 +53,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyByteArray_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -84,11 +83,10 @@
             ("mutable", "METH_NOARGS",
              """
                 PyObject *base;
-                char * p_str;
                 base = PyByteArray_FromStringAndSize("test", 10);
                 if (PyByteArray_GET_SIZE(base) != 10)
                     return PyLong_FromLong(-PyByteArray_GET_SIZE(base));
-                memcpy(PyByteArray_AS_STRING(base), "works", 6); 
+                memcpy(PyByteArray_AS_STRING(base), "works", 6);
                 Py_INCREF(base);
                 return base;
              """),
@@ -141,9 +139,9 @@
             ("concat", "METH_VARARGS",
              """
                 PyObject * ret, *right, *left;
-                PyObject *ba1, *ba2; 
+                PyObject *ba1, *ba2;
                 if (!PyArg_ParseTuple(args, "OO", &left, &right)) {
-                    return PyString_FromString("parse failed"); 
+                    return PyString_FromString("parse failed");
                 }
                 ba1 = PyByteArray_FromObject(left);
                 ba2 = PyByteArray_FromObject(right);
@@ -171,9 +169,9 @@
              PyObject *obj, *ba;
              int newsize, oldsize, ret;
              if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) {
-                 return PyString_FromString("parse failed"); 
+                 return PyString_FromString("parse failed");
              }
-             
+
              ba = PyByteArray_FromObject(obj);
              if (ba == NULL)
                  return NULL;
@@ -187,7 +185,7 @@
              {
                   printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize);
                   return NULL;
-             } 
+             }
              return ba;
              '''
             )])
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -40,7 +40,7 @@
                  #endif
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
-                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); 
+                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -49,7 +49,7 @@
             ("test_Size_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyString_Size(f);
+                 PyString_Size(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -72,7 +72,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyString_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -100,7 +99,6 @@
                 PyObject *base;
                 PyTypeObject * type;
                 PyStringObject *obj;
-                char * p_str;
                 base = PyString_FromString("test");
                 if (PyString_GET_SIZE(base) != 4)
                     return PyLong_FromLong(-PyString_GET_SIZE(base));
@@ -312,17 +310,17 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyStringObject*)obj)->ob_shash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ("test_sstate", "METH_NOARGS",
              '''
                 PyObject *s = PyString_FromString("xyz");
-                int sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*int sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 PyString_InternInPlace(&s);
-                sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 Py_DECREF(s);
                 return PyBool_FromLong(1);
              '''),
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -8,7 +8,7 @@
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import platform
+from rpython.translator.platform.distutils_platform import DistutilsPlatform
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -18,6 +18,8 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
+rpy_platform = DistutilsPlatform()
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -50,7 +52,7 @@
         files = convert_sources_to_files(source_strings, dirname)
         source_files = files
     eci = ExternalCompilationInfo(include_dirs=include_dirs, **kwds)
-    soname = platform.platform.compile(source_files, eci,
+    soname = rpy_platform.compile(source_files, eci,
         outputfilename=str(dirname/modname),
         standalone=False)
     return soname
@@ -83,7 +85,7 @@
     else:
         kwds["link_files"] = [str(api_library + '.so')]
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"] = ["-Werror", "-g", "-O0"]
+            kwds["compile_extra"] = ["-Werror", "-g", "-O0", "-fPIC"]
             kwds["link_extra"] = ["-g"]
 
     modname = modname.split('.')[-1]
@@ -116,7 +118,8 @@
     elif sys.platform == 'darwin':
         pass
     elif sys.platform.startswith('linux'):
-        kwds["compile_extra"] = ["-O0", "-g","-Werror=implicit-function-declaration"]
+        kwds["compile_extra"] = [
+            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
 
     modname = modname.split('.')[-1]
     soname = create_so(modname,
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py
--- a/pypy/module/cpyext/test/test_frameobject.py
+++ b/pypy/module/cpyext/test/test_frameobject.py
@@ -13,7 +13,7 @@
                  PyObject *empty_string = PyString_FromString("");
                  PyObject *empty_tuple = PyTuple_New(0);
                  PyCodeObject *py_code;
-                 PyFrameObject *py_frame;
+                 PyFrameObject *py_frame = NULL;
 
                  py_code = PyCode_New(
                      0,            /*int argcount,*/
@@ -75,7 +75,7 @@
              """
                  int check;
                  PyObject *type, *value, *tb;
-                 PyObject *ret = PyRun_String("XXX", Py_eval_input, 
+                 PyObject *ret = PyRun_String("XXX", Py_eval_input,
                                               Py_None, Py_None);
                  if (ret) {
                      Py_DECREF(ret);
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -149,7 +149,6 @@
         pybuffer = self.import_parser(
             '''
             Py_buffer buf1, buf2, buf3;
-            PyObject *result;
             if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) {
                 return NULL;
             }
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -127,12 +127,12 @@
         test_compare(1, 2)
         test_compare(2, 2)
         test_compare('2', '1')
-        
+
         w_i = space.wrap(1)
         assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1
         assert api.PyErr_Occurred() is space.w_SystemError
         api.PyErr_Clear()
-        
+
     def test_IsInstance(self, space, api):
         assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1
         assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0
@@ -165,7 +165,7 @@
             return File""")
         w_f = space.call_function(w_File)
         assert api.PyObject_AsFileDescriptor(w_f) == 42
-    
+
     def test_hash(self, space, api):
         assert api.PyObject_Hash(space.wrap(72)) == 72
         assert api.PyObject_Hash(space.wrap(-1)) == -1
@@ -250,7 +250,7 @@
                  if (copy != orig)
                      PyObject_Free(copy);
                  PyObject_Free(orig);
-                 return ret;  
+                 return ret;
              """)])
         x = module.realloctest()
         assert x == 'hello world\x00'
@@ -425,7 +425,6 @@
                  """
     Py_buffer buf;
     PyObject *str = PyString_FromString("hello, world.");
-    PyObject *result;
 
     if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13,
                           1, PyBUF_WRITABLE)) {
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -130,7 +130,7 @@
         module = self.import_extension('foo', [
             ("run", "METH_NOARGS",
              """
-                long prev, next;
+                long prev;
                 PyObject *t = PyTuple_New(1);
                 prev = Py_True->ob_refcnt;
                 Py_INCREF(Py_True);
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -737,7 +737,6 @@
              """
                 IntLikeObject *intObj;
                 int intval;
-                PyObject *name;
 
                 if (!PyArg_ParseTuple(args, "i", &intval))
                     return NULL;
@@ -1058,7 +1057,6 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                PyObject *obj;
                 FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
                 FooType_Type.tp_base = &PyType_Type;
                 if (PyType_Ready(&FooType_Type) < 0) return NULL;
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -35,7 +35,7 @@
             ("test_GetSize_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyUnicode_GetSize(f);
+                 PyUnicode_GetSize(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -57,7 +57,6 @@
              """
                  PyObject *s, *t;
                  Py_UNICODE* c;
-                 Py_ssize_t len;
 
                  s = PyUnicode_FromUnicode(NULL, 4);
                  if (s == NULL)
@@ -84,7 +83,7 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyUnicodeObject*)obj)->hash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ])
@@ -234,13 +233,13 @@
         w_res = api.PyUnicode_AsUTF8String(w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\tm'
-    
+
     def test_decode_utf8(self, space, api):
         u = rffi.str2charp(u'sp\x134m'.encode("utf-8"))
         w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp\x134m'
-        
+
         w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
@@ -405,7 +404,7 @@
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
         result = api.PyUnicode_AsASCIIString(w_ustr)
-        
+
         assert space.eq_w(space.wrap(ustr), result)
 
         w_ustr = space.wrap(u"abcd\xe9f")

From pypy.commits at gmail.com  Tue Jun 14 15:40:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 12:40:46 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: jump_in_time(exact=False)
Message-ID: <57605dbe.49c51c0a.8f92b.1347@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85168:e5d6d6128ad7
Date: 2016-06-14 21:41 +0200
http://bitbucket.org/pypy/pypy/changeset/e5d6d6128ad7/

Log:	jump_in_time(exact=False)

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -53,14 +53,16 @@
     _change_time('f', time_delta, callback, arg_string)
 
 @specialize.arg(1)
-def jump_in_time(target_time, callback, arg_string):
+def jump_in_time(target_time, callback, arg_string, exact=True):
     """For RPython debug commands: the debugger should run the
     'go ' command.  This will reset the memory and fork again,
     so you can't save any RPython state and read it back.  You can only
     encode the state you want to save into a string.  In the reloaded
-    process, 'callback(arg_string)' is called.
+    process, 'callback(arg_string)' is called.  If 'exact' is False, go to
+    the fork point before target_time but don't go_forward to exactly
+    target_time afterwards.
     """
-    _change_time('g', target_time, callback, arg_string)
+    _change_time('g' if exact else 'b', target_time, callback, arg_string)
 
 def currently_created_objects():
     """For RPython debug commands: returns the current value of
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -233,6 +233,7 @@
 
 struct jump_in_time_s {
     uint64_t target_time;
+    char mode;
     void *callback;
     size_t arg_length;
 };
@@ -531,11 +532,12 @@
 }
 
 static void cmd_go(uint64_t target_time, void callback(RPyString *),
-                   RPyString *arg)
+                   RPyString *arg, char mode)
 {
     struct jump_in_time_s header;
 
     header.target_time = target_time;
+    header.mode = mode;
     header.callback = callback;    /* may be NULL */
     /* ^^^ assumes the fn address is the same in the various forks */
     header.arg_length = arg == NULL ? 0 : RPyString_Size(arg);
@@ -556,7 +558,7 @@
 
     if (process_kind == PK_DEBUG_PROCESS) {
         printf("At end.\n");
-        cmd_go(stop_points, NULL, NULL);
+        cmd_go(stop_points, NULL, NULL, 'g');
         abort();   /* unreachable */
     }
 
@@ -641,7 +643,13 @@
             process_kind = PK_DEBUG_PROCESS;
             assert(jump_in_time.target_time >= rpy_revdb.stop_point_seen);
             most_recent_fork = rpy_revdb.stop_point_seen;
-            rpy_revdb.stop_point_break = jump_in_time.target_time;
+            switch (jump_in_time.mode) {
+            case 'b':    /* go non-exact: stay at most_recent_fork */
+                rpy_revdb.stop_point_break = most_recent_fork;
+                break;
+            default:     /* other modes: go exact */
+                rpy_revdb.stop_point_break = jump_in_time.target_time;
+            }
 
             if (jump_in_time.callback == NULL) {
                 assert(jump_in_time.arg_length == 0);
@@ -671,7 +679,7 @@
                 ;     /* normal exit */
             else {
                 fprintf(stderr, "debugging subprocess died\n");
-                cmd_go((uint64_t)-1, NULL, NULL);
+                cmd_go((uint64_t)-1, NULL, NULL, 'q');
                 abort();    /* unreachable */
             }
         }
@@ -736,7 +744,7 @@
 
 static void act_quit(char *p)
 {
-    cmd_go((uint64_t)-1, NULL, NULL);
+    cmd_go((uint64_t)-1, NULL, NULL, 'q');
 }
 
 static void act_go(char *p)
@@ -746,7 +754,7 @@
         printf("usage: go \n");
         return;
     }
-    cmd_go(target_time, NULL, NULL);
+    cmd_go(target_time, NULL, NULL, 'g');
 }
 
 static void act_info(char *p)
@@ -844,10 +852,10 @@
         invoke_argument = arg;
         break;
     }
-    case 'g': {      /* go */
-        cmd_go(time >= 1 ? time : 1, callback, arg);
+    case 'g':       /* go */
+    case 'b':       /* go non exact */
+        cmd_go(time >= 1 ? time : 1, callback, arg, mode);
         abort();    /* unreachable */
-    }
     default:
         abort();    /* unreachable */
     }
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -334,6 +334,8 @@
                 revdb.go_forward(1, went_fw, "xx")
             if cmdline == 'change-time':
                 revdb.jump_in_time(2, changed_time, "xyzzy")
+            if cmdline == 'change-time-non-exact':
+                revdb.jump_in_time(2, changed_time, "nonx", exact=False)
             if cmdline == 'set-break-after-0':
                 dbstate.break_after = 0
             if cmdline == 'print-id':
@@ -442,6 +444,16 @@
                       'went-fw zz -> 3\r\n'
                       '(3)$ ')
 
+    def test_change_time_non_exact(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r change-time-non-exact')
+        child.expectx('<<>>\r\n'
+                      'changed-time nonx -> 1\r\n'
+                      'went-fw zz -> 2\r\n'
+                      'went-fw yy -> 3\r\n'
+                      '(3)$ ')
+
     def test_dynamic_breakpoint(self):
         py.test.skip("unsure if that's needed")
         child = self.replay()

From pypy.commits at gmail.com  Tue Jun 14 16:31:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 13:31:40 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: finish and test object
 tracking in detail
Message-ID: <576069ac.a91cc20a.d7d7b.fffff69d@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85169:5527e8d6a710
Date: 2016-06-14 22:32 +0200
http://bitbucket.org/pypy/pypy/changeset/5527e8d6a710/

Log:	finish and test object tracking in detail

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -81,14 +81,15 @@
     """
     return llop.revdb_get_unique_id(lltype.SignedLongLong, x)
 
-def track_objects(unique_id):
+def track_object(unique_id):
     """Track the creation of the object given by its unique_id, which must
     be in the future (i.e. >= currently_created_objects()).  Call this
     before go_forward().  If go_forward() goes over the creation of this
     object, then afterwards, get_tracked_object() returns the object.
     Going forward is also interrupted at the following stop point.
+    Object tracking is lost by jump_in_time(), like everything else.
     """
-    return llop.revdb_track_object(lltype.Bool, x)
+    llop.revdb_track_object(lltype.Void, unique_id)
 
 @specialize.arg(0)
 def get_tracked_object(Class=llmemory.GCREF):   # or an RPython class
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -370,13 +370,15 @@
 
 static void enable_io(rpy_revdb_t *dinfo)
 {
-    uint64_t v;
+    uint64_t v1, v2;
     flag_io_disabled = 0;
 
-    /* restore the complete struct, with the exception of 'stop_point_break' */
-    v = rpy_revdb.stop_point_break;
+    /* restore the complete struct, with the exception of '*_break' */
+    v1 = rpy_revdb.stop_point_break;
+    v2 = rpy_revdb.unique_id_break;
     rpy_revdb = *dinfo;
-    rpy_revdb.stop_point_break = v;
+    rpy_revdb.stop_point_break = v1;
+    rpy_revdb.unique_id_break = v2;
 }
 
 /* generated by RPython */
@@ -880,12 +882,40 @@
     }
 }
 
+static void *tracked_object;
+
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
-    fprintf(stderr, "PING\n");
-    abort();
+    tracked_object = new_object;
+    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
+    return rpy_revdb.unique_id_seen;
 }
 
+RPY_EXTERN
+void rpy_reverse_db_track_object(long long unique_id)
+{
+    if (stopped_uid <= 0) {
+        fprintf(stderr, "stopped_uid should not be <= 0\n");
+        exit(1);
+    }
+    if (unique_id <= 0) {
+        printf("cannot track a prebuilt or debugger-created object\n");
+        return;
+    }
+    if (unique_id < stopped_uid) {
+        printf("cannot track the creation of an object already created\n");
+        return;
+    }
+    rpy_revdb.unique_id_break = unique_id;
+    tracked_object = NULL;
+}
+
+RPY_EXTERN
+void *rpy_reverse_db_get_tracked_object(void)
+{
+    return tracked_object;
+}
+
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -96,6 +96,12 @@
 #define OP_REVDB_GET_UNIQUE_ID(x, r)                                    \
     r = ((struct pypy_header0 *)x)->h_uid
 
+#define OP_REVDB_TRACK_OBJECT(uid, r)                                   \
+    rpy_reverse_db_track_object(uid)
+
+#define OP_REVDB_GET_TRACKED_OBJECT(r)                                  \
+    r = rpy_reverse_db_get_tracked_object()
+
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
                                       const char *file, int line);
@@ -107,6 +113,8 @@
                                            RPyString *arg);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
+RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id);
+RPY_EXTERN void *rpy_reverse_db_get_tracked_object(void);
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -339,13 +339,17 @@
             if cmdline == 'set-break-after-0':
                 dbstate.break_after = 0
             if cmdline == 'print-id':
-                revdb.send_output('%d %d\n' % (
+                revdb.send_output('obj.x=%d %d %d\n' % (
+                    dbstate.stuff.x,
                     revdb.get_unique_id(dbstate.stuff),
                     revdb.currently_created_objects()))
-            #if cmdline.startswith('check-id '):
-            #    obj_id = int(cmdline[len('check-id '):])
-            #    revdb.send_output("%d\n" %
-            #        int(revdb.id_to_object(Stuff, obj_id) is dbstate.stuff))
+            if cmdline.startswith('track-object '):
+                uid = int(cmdline[len('track-object '):])
+                revdb.track_object(uid)
+            if cmdline == 'get-tracked-object':
+                obj = revdb.get_tracked_object(Stuff)
+                revdb.send_output('None\n' if obj is None else
+                                  ('obj.x=%d\n' % obj.x))
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -466,24 +470,96 @@
         child.expectx('breakpoint!\r\n'
                       '(2)$ ')
 
-    def test_get_unique_id(self):
+    def test_get_unique_id_and_track_object(self):
         child = self.replay()
         child.expectx('(3)$ ')
         child.sendline('r print-id')
         child.expect(re.escape('<<>>\r\n')
-                     + r'(\d+) (\d+)'
+                     + r'obj.x=1002 (\d+) (\d+)'
                      + re.escape('\r\n'
                                  'blipped\r\n'
                                  '(3)$ '))
-        object_id = int(child.match.group(1))
-        currenty_created_objects = int(child.match.group(2))
-        assert 0 < object_id < currenty_created_objects
-        XXX
-        for at_time in [1, 2, 3]:
-            child.sendline('__go %d' % at_time)
-            child.expectx('(%d)$ ' % at_time)
-            child.sendline('r check-id ' + object_id)
-            child.expectx('<<>>\r\n' % (object_id,) +
-                          '1\r\n' +
-                          'blipped\r\n' +
-                          '(%d)$ ' % at_time)
+        object_id_3rd = int(child.match.group(1))
+        created_objects_3rd = int(child.match.group(2))
+        assert 0 < object_id_3rd < created_objects_3rd
+        #
+        child.sendline('__go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r print-id')
+        child.expect(re.escape('<<>>\r\n')
+                     + r'obj.x=1000 (\d+) (\d+)'
+                     + re.escape('\r\n'
+                                 'blipped\r\n'
+                                 '(1)$ '))
+        object_id_1st = int(child.match.group(1))
+        created_objects_1st = int(child.match.group(2))
+        assert 0 < object_id_1st < created_objects_1st
+        assert created_objects_1st <= object_id_3rd   # only created afterwards
+        #
+        child.sendline('r track-object %d' % object_id_3rd)
+        child.expectx('<<>>\r\n' % object_id_3rd +
+                      'blipped\r\n'
+                      '(1)$ ')
+        prev_time = 1
+        for i in [1, 2, 3]:
+            child.sendline('r get-tracked-object')
+            child.expectx('<<>>\r\n'
+                          'None\r\n'
+                          'blipped\r\n'
+                          '(%d)$ ' % prev_time)
+            child.sendline('__forward %d' % (i - prev_time))
+            child.expectx('(%d)$ ' % i)
+            prev_time = i
+            child.sendline('r print-id')
+            child.expect(re.escape('<<>>\r\n')
+                         + r'obj.x=%d (\d+) (\d+)' % (1000 + prev_time - 1)
+                         + re.escape('\r\n'
+                                     'blipped\r\n'
+                                     '(%d)$ ' % prev_time))
+        child.sendline('r get-tracked-object')
+        child.expectx('<<>>\r\n'
+                      'obj.x=1002\r\n'
+                      'blipped\r\n'
+                      '(3)$ ')
+        child.sendline('__go 3')
+        child.expectx('(3)$ ')
+        child.sendline('r get-tracked-object')
+        child.expectx('<<>>\r\n'
+                      'None\r\n'
+                      'blipped\r\n'
+                      '(3)$ ')
+        #
+        child.sendline('__go 2')
+        child.expectx('(2)$ ')
+        child.sendline('r print-id')
+        child.expect(re.escape('<<>>\r\n')
+                     + r'obj.x=1001 (\d+) (\d+)'
+                     + re.escape('\r\n'
+                                 'blipped\r\n'
+                                 '(2)$ '))
+        object_id_2nd = int(child.match.group(1))
+        created_objects_2nd = int(child.match.group(2))
+        #
+        child.sendline('r track-object %d' % object_id_2nd)
+        child.expectx('<<>>\r\n' % object_id_2nd +
+                    'cannot track the creation of an object already created\r\n'
+                      'blipped\r\n'
+                      '(2)$ ')
+        child.sendline('r track-object 0')
+        child.expectx('<<>>\r\n'
+                      'cannot track a prebuilt or debugger-created object\r\n'
+                      'blipped\r\n'
+                      '(2)$ ')
+        child.sendline('__go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r track-object %d' % object_id_2nd)
+        child.expectx('<<>>\r\n' % object_id_2nd +
+                      'blipped\r\n'
+                      '(1)$ ')
+        child.sendline('__forward 5')
+        child.expectx('(2)$ ')
+        child.sendline('r get-tracked-object')
+        child.expectx('<<>>\r\n'
+                      'obj.x=1001\r\n'
+                      'blipped\r\n'
+                      '(2)$ ')

From pypy.commits at gmail.com  Tue Jun 14 16:32:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Jun 2016 13:32:06 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <576069c6.e4b3c20a.fc515.ffffe3b5@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r756:4da1183a196d
Date: 2016-06-14 22:33 +0200
http://bitbucket.org/pypy/pypy.org/changeset/4da1183a196d/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 
 
    
-   $64088 of $105000 (61.0%)
+   $64178 of $105000 (61.1%)
    
    
    
    
 
@@ -23,7 +23,7 @@
   
  • 
   
   
    • 
   
  • 

From pypy.commits at gmail.com  Tue Jun 14 18:09:28 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 15:09:28 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: allow subclassing
 PyUnicode_Type
Message-ID: <57608098.82e01c0a.96566.3b2c@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85170:752495b32c5f
Date: 2016-06-14 21:18 +0300
http://bitbucket.org/pypy/pypy/changeset/752495b32c5f/

Log:	allow subclassing PyUnicode_Type

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -77,7 +77,9 @@
     """
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
     s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length)
-    w_obj = space.wrap(s)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
+    w_obj.__init__(s)
     py_uni.c_hash = space.hash_w(w_obj)
     track_reference(space, py_obj, w_obj)
     return w_obj

From pypy.commits at gmail.com  Tue Jun 14 18:09:30 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 15:09:30 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: convert PyStringObject
 from using a char * buffer to using a char[], also fixup type itemsize
Message-ID: <5760809a.41561c0a.9645d.27d7@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85171:7d69e9f87f9c
Date: 2016-06-14 23:59 +0300
http://bitbucket.org/pypy/pypy/changeset/7d69e9f87f9c/

Log:	convert PyStringObject from using a char * buffer to using a char[],
	also fixup type itemsize

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -18,7 +18,7 @@
 ## -----------
 ##
 ## PyString_AsString() must return a (non-movable) pointer to the underlying
-## buffer, whereas pypy strings are movable.  C code may temporarily store
+## ob_sval, whereas pypy strings are movable.  C code may temporarily store
 ## this address and use it, as long as it owns a reference to the PyObject.
 ## There is no "release" function to specify that the pointer is not needed
 ## any more.
@@ -30,7 +30,7 @@
 ## --------
 ##
 ## PyStringObject contains two additional members: the ob_size and a pointer to a
-## char buffer; it may be NULL.
+## char ob_sval; it may be NULL.
 ##
 ## - A string allocated by pypy will be converted into a PyStringObject with a
 ##   NULL buffer.  The first time PyString_AsString() is called, memory is
@@ -58,7 +58,7 @@
 PyStringObjectStruct = lltype.ForwardReference()
 PyStringObject = lltype.Ptr(PyStringObjectStruct)
 PyStringObjectFields = PyVarObjectFields + \
-    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP))
+    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char)))
 cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct)
 
 @bootstrap_function
@@ -74,8 +74,8 @@
 
 def new_empty_str(space, length):
     """
-    Allocate a PyStringObject and its buffer, but without a corresponding
-    interpreter object.  The buffer may be mutated, until string_realize() is
+    Allocate a PyStringObject and its ob_sval, but without a corresponding
+    interpreter object.  The ob_sval may be mutated, until string_realize() is
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_str.layout.typedef)
@@ -84,48 +84,45 @@
 
     buflen = length + 1
     py_str.c_ob_size = length
-    py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
 def string_attach(space, py_obj, w_obj):
     """
-    Fills a newly allocated PyStringObject with the given string object. The
-    buffer must not be modified.
+    Copy RPython string object contents to a PyStringObject. The
+    c_ob_sval must not be modified.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    py_str.c_ob_size = len(space.str_w(w_obj))
-    py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
+    s = space.str_w(w_obj)
+    if py_str.c_ob_size  <= len(s):
+        raise oefmt(space.w_ValueError,
+            "string_attach called on object with ob_size %d but trying to store %d",
+            py_str.c_ob_size, len(s) + 1) 
+    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def string_realize(space, py_obj):
     """
-    Creates the string in the interpreter. The PyStringObject buffer must not
+    Creates the string in the interpreter. The PyStringObject ob_sval must not
     be modified after this call.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    if not py_str.c_buffer:
-        py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                    flavor='raw', zero=True)
-    s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
+    s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size)
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(W_BytesObject, w_type)
     w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
+    print 'string_realize',s,py_str.c_ob_size
     return w_obj
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def string_dealloc(space, py_obj):
     """Frees allocated PyStringObject resources.
     """
-    py_str = rffi.cast(PyStringObject, py_obj)
-    if py_str.c_buffer:
-        lltype.free(py_str.c_buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -161,12 +158,10 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    return ref_str.c_buffer
+    if not pyobj_has_w_obj(ref):
+        # XXX Force the ref?
+        string_realize(space, ref)
+    return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
 def PyString_AS_STRING(space, void_ref):
@@ -175,14 +170,11 @@
     # return the c-level ptr as RW
     if not pyobj_has_w_obj(ref):
         py_str = rffi.cast(PyStringObject, ref)
-        if not py_str.c_buffer:
-            py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                        flavor='raw', zero=True)
-        return py_str.c_buffer
+        return py_str.c_ob_sval
     return _PyString_AsString(space, ref)
 
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
-def PyString_AsStringAndSize(space, ref, buffer, length):
+def PyString_AsStringAndSize(space, ref, data, length):
     if not PyString_Check(space, ref):
         from pypy.module.cpyext.unicodeobject import (
             PyUnicode_Check, _PyUnicode_AsDefaultEncodedString)
@@ -192,18 +184,16 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
+    if not pyobj_has_w_obj(ref):
+        # force the ref
+        string_realize(space, ref)
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    buffer[0] = ref_str.c_buffer
+    data[0] = ref_str.c_ob_sval
     if length:
         length[0] = ref_str.c_ob_size
     else:
         i = 0
-        while ref_str.c_buffer[i] != '\0':
+        while ref_str.c_ob_sval[i] != '\0':
             i += 1
         if i != ref_str.c_ob_size:
             raise oefmt(space.w_TypeError,
@@ -214,7 +204,7 @@
 def PyString_Size(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         ref = rffi.cast(PyStringObject, ref)
-        return ref.c_ob_size
+        return ref.c_ob_size - 1
     else:
         w_obj = from_ref(space, ref)
         return space.len_w(w_obj)
@@ -233,7 +223,7 @@
     """
     # XXX always create a new string so far
     py_str = rffi.cast(PyStringObject, ref[0])
-    if not py_str.c_buffer:
+    if pyobj_has_w_obj(py_str):
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
     try:
@@ -247,7 +237,7 @@
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newstr.c_buffer[i] = py_str.c_buffer[i]
+        py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newstr)
     return 0
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -40,12 +40,11 @@
     PyObject_VAR_HEAD
     long ob_shash;
     int ob_sstate;
-    char * buffer; /* change the name from cpython so all non-api c access is thwarted */
+    char ob_sval[1]; 
 
     /* Invariants 
-     * (not relevant in PyPy, all stringobjects are backed by a pypy object)
-     *     buffer contains space for 'ob_size+1' elements.
-     *     buffer[ob_size] == 0.
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
      *     ob_shash is the hash of the string or -1 if not computed yet.
      *     ob_sstate != 0 iff the string object is in stringobject.c's
      *       'interned' dictionary; in this case the two references
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    INTERPLEVEL_API)
+    INTERPLEVEL_API, PyVarObject)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.objectobject import W_ObjectObject
@@ -47,13 +47,16 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount and w_type is not space.w_str:
+        if itemcount:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
+        if itemcount:
+            pyvarobj = rffi.cast(PyVarObject, pyobj)
+            pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
         #pyobj.c_ob_pypy_link should get assigned very quickly
         pyobj.c_ob_type = pytype
@@ -152,13 +155,18 @@
 class InvalidPointerException(Exception):
     pass
 
-def create_ref(space, w_obj, itemcount=0):
+def create_ref(space, w_obj):
     """
     Allocates a PyObject, and fills its fields with info from the given
     interpreter object.
     """
     w_type = space.type(w_obj)
+    pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
+    if pytype.c_tp_itemsize != 0:
+        itemcount = space.len_w(w_obj) + 1 # PyStringObject and subclasses
+    else:
+        itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)
     track_reference(space, py_obj, w_obj)
     #
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -25,14 +25,13 @@
             ("test_Size", "METH_NOARGS",
              """
                  PyObject* s = PyString_FromString("Hello world");
-                 int result = 0;
+                 int result;
                  size_t expected_size;
 
-                 if(PyString_Size(s) == 11) {
-                     result = 1;
-                 }
+                 result = PyString_Size(s);
+                
                  #ifdef PYPY_VERSION
-                    expected_size = sizeof(void*)*7;
+                    expected_size = 48;
                  #elif defined Py_DEBUG
                     expected_size = 53;
                  #else
@@ -44,7 +43,7 @@
                      result = 0;
                  }
                  Py_DECREF(s);
-                 return PyBool_FromLong(result);
+                 return PyLong_FromLong(result);
              """),
             ("test_Size_exception", "METH_NOARGS",
              """
@@ -60,7 +59,7 @@
              """)], prologue='#include ')
         assert module.get_hello1() == 'Hello world'
         assert module.get_hello2() == 'Hello world'
-        assert module.test_Size()
+        assert module.test_Size() == 11
         raises(TypeError, module.test_Size_exception)
 
         assert module.test_is_string("")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -695,6 +695,8 @@
     if pto.c_tp_base:
         if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize:
             pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize
+        if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
+            pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
     # will be filled later on with the correct value
     # may not be 0

From pypy.commits at gmail.com  Tue Jun 14 18:09:33 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 15:09:33 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: only use AS_STRING for
 non-interned access
Message-ID: <5760809d.4aa71c0a.9b4b8.3ebf@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85173:ed79f1e7c24f
Date: 2016-06-15 00:48 +0300
http://bitbucket.org/pypy/pypy/changeset/ed79f1e7c24f/

Log:	only use AS_STRING for non-interned access

diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c
--- a/pypy/module/cpyext/src/stringobject.c
+++ b/pypy/module/cpyext/src/stringobject.c
@@ -107,7 +107,7 @@
     if (!string)
         return NULL;
 
-    s = PyString_AsString(string);
+    s = PyString_AS_STRING(string);
 
     for (f = format; *f; f++) {
         if (*f == '%') {
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -80,7 +80,7 @@
                  if (t == NULL)
                     return NULL;
                  Py_DECREF(t);
-                 c = PyString_AsString(s);
+                 c = PyString_AS_STRING(s);
                  c[0] = 'a';
                  c[1] = 'b';
                  c[2] = 0;
@@ -445,22 +445,22 @@
     def test_string_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_buffer[0] = 'a'
-        py_str.c_buffer[1] = 'b'
-        py_str.c_buffer[2] = 'c'
+        py_str.c_sval[0] = 'a'
+        py_str.c_sval[1] = 'b'
+        py_str.c_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 3)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[3] == '\x00'
+        assert py_str.c_sval[1] == 'b'
+        assert py_str.c_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 10)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[10] == '\x00'
+        assert py_str.c_sval[1] == 'b'
+        assert py_str.c_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 

From pypy.commits at gmail.com  Tue Jun 14 18:09:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 15:09:35 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: use length in new_str,
 fix typos in test
Message-ID: <5760809f.692dc20a.489bc.06d4@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85174:737eb2235db4
Date: 2016-06-15 01:06 +0300
http://bitbucket.org/pypy/pypy/changeset/737eb2235db4/

Log:	use length in new_str, fix typos in test

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -79,11 +79,9 @@
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_str.layout.typedef)
-    py_obj = typedescr.allocate(space, space.w_str)
+    py_obj = typedescr.allocate(space, space.w_str, length)
     py_str = rffi.cast(PyStringObject, py_obj)
-
-    buflen = length + 1
-    py_str.c_ob_size = length
+    py_str.c_ob_shash = -1
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
@@ -223,7 +221,6 @@
     # XXX always create a new string so far
     py_str = rffi.cast(PyStringObject, ref[0])
     if pyobj_has_w_obj(py_str):
-        import pdb;pdb.set_trace()
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
     try:
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -445,22 +445,22 @@
     def test_string_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_sval[0] = 'a'
-        py_str.c_sval[1] = 'b'
-        py_str.c_sval[2] = 'c'
+        py_str.c_ob_sval[0] = 'a'
+        py_str.c_ob_sval[1] = 'b'
+        py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 3)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_sval[1] == 'b'
-        assert py_str.c_sval[3] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 10)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_sval[1] == 'b'
-        assert py_str.c_sval[10] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 

From pypy.commits at gmail.com  Tue Jun 14 18:09:32 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 14 Jun 2016 15:09:32 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: rework c_ob_size,
 it is len(w_str) but the size of c_ob_sval is c_ob_size
Message-ID: <5760809c.2523c20a.e0232.07cc@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85172:098123589947
Date: 2016-06-15 00:48 +0300
http://bitbucket.org/pypy/pypy/changeset/098123589947/

Log:	rework c_ob_size, it is len(w_str) but the size of c_ob_sval is
	c_ob_size

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -94,10 +94,10 @@
     """
     py_str = rffi.cast(PyStringObject, py_obj)
     s = space.str_w(w_obj)
-    if py_str.c_ob_size  <= len(s):
+    if py_str.c_ob_size  < len(s):
         raise oefmt(space.w_ValueError,
             "string_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s) + 1) 
+            py_str.c_ob_size, len(s)) 
     rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
@@ -116,7 +116,6 @@
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
-    print 'string_realize',s,py_str.c_ob_size
     return w_obj
 
 @cpython_api([PyObject], lltype.Void, header=None)
@@ -204,7 +203,7 @@
 def PyString_Size(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         ref = rffi.cast(PyStringObject, ref)
-        return ref.c_ob_size - 1
+        return ref.c_ob_size
     else:
         w_obj = from_ref(space, ref)
         return space.len_w(w_obj)
@@ -224,6 +223,7 @@
     # XXX always create a new string so far
     py_str = rffi.cast(PyStringObject, ref[0])
     if pyobj_has_w_obj(py_str):
+        import pdb;pdb.set_trace()
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
     try:
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -47,14 +47,14 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount:
+        if pytype.c_tp_itemsize:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
-        if itemcount:
+        if pytype.c_tp_itemsize:
             pyvarobj = rffi.cast(PyVarObject, pyobj)
             pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
@@ -164,7 +164,7 @@
     pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
     if pytype.c_tp_itemsize != 0:
-        itemcount = space.len_w(w_obj) + 1 # PyStringObject and subclasses
+        itemcount = space.len_w(w_obj) # PyStringObject and subclasses
     else:
         itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)

From pypy.commits at gmail.com  Wed Jun 15 04:16:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Jun 2016 01:16:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Missing decref in error case
Message-ID: <57610ee0.86c71c0a.d7260.ffffdf40@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r85175:5ea0348fcbee
Date: 2016-06-15 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/5ea0348fcbee/

Log:	Missing decref in error case

diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference

From pypy.commits at gmail.com  Wed Jun 15 07:40:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Jun 2016 04:40:41 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: - Fix link for linux32
Message-ID: <57613eb9.89acc20a.65989.3c8a@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r757:4bea80355a18
Date: 2016-06-15 13:42 +0200
http://bitbucket.org/pypy/pypy.org/changeset/4bea80355a18/

Log:	- Fix link for linux32

	- Fix Ubuntu version for pypy3 on linux32

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -92,7 +92,7 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux32.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux64.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raring.tar.bz2
@@ -116,7 +116,7 @@
     Warning: this is an alpha release supporting the Python 3.3 language.
     It's also known to be (sometimes much) slower than PyPy 2.
 
-* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS)`__ (see ``[1]`` below)
+* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below)
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)

From pypy.commits at gmail.com  Wed Jun 15 07:46:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Jun 2016 04:46:24 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57614010.e153c20a.8d0cb.0ec3@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r758:69f7b34a0213
Date: 2016-06-15 13:47 +0200
http://bitbucket.org/pypy/pypy.org/changeset/69f7b34a0213/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 
 
    
-   $64178 of $105000 (61.1%)
+   $64596 of $105000 (61.5%)
    
    
    
    
 
@@ -23,7 +23,7 @@
   
  • 
   
   
    • 
   
  • 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    
-   $30732 of $80000 (38.4%)
+   $30744 of $80000 (38.4%)
    
    
    
    
 
@@ -25,7 +25,7 @@
   
  • 
   
   
    • 
   
  • 

From pypy.commits at gmail.com  Wed Jun 15 08:52:38 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 15 Jun 2016 05:52:38 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: detect altivec at runtime on
 linux
Message-ID: <57614f96.0654c20a.1573.1c12@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85176:ebfa9b3a75d6
Date: 2016-06-15 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/ebfa9b3a75d6/

Log:	detect altivec at runtime on linux

diff --git a/rpython/jit/backend/ppc/detect_feature.py b/rpython/jit/backend/ppc/detect_feature.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/detect_feature.py
@@ -0,0 +1,37 @@
+import sys
+import struct
+import platform
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
+from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rstruct.runpack import runpack
+
+AT_HWCAP = rffi_platform.getconstantinteger('AT_HWCAP', '#include "linux/auxvec.h"')
+AT_NULL = rffi_platform.getconstantinteger('AT_NULL', '#include "linux/auxvec.h"')
+PPC_FEATURE_HAS_ALTIVEC = rffi_platform.getconstantinteger('PPC_FEATURE_HAS_ALTIVEC',
+                                                   '#include "asm/cputable.h"')
+SYSTEM = platform.system()
+
+def detect_vsx_linux():
+    with open('/proc/self/auxv', 'rb') as fd:
+        while True:
+            buf = fd.read(16)
+            if not buf:
+                break
+            key, value = runpack("LL", buf)
+            if key == AT_HWCAP:
+                if value & PPC_FEATURE_HAS_ALTIVEC:
+                    return True
+            if key == AT_NULL:
+                return False
+    return False
+
+def detect_vsx():
+    if SYSTEM == 'Linux':
+        return detect_vsx_linux()
+    return False
+
+if __name__ == '__main__':
+    print 'The following extensions are supported:'
+    if detect_vsx():
+        print '  - AltiVec'
diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -11,6 +11,10 @@
 
 class PPC_CPU(AbstractLLCPU):
 
+    vector_extension = False # may be set to true in setup
+    vector_register_size = 16
+    vector_horizontal_operations = False
+
     supports_floats = True
     # missing: supports_singlefloats
 
@@ -38,6 +42,9 @@
 
     def setup(self):
         self.assembler = AssemblerPPC(self)
+        if detect_vsx():
+            self.vector_extension = True
+            # ??? self.vector_horizontal_operations = True
 
     @rgc.no_release_gil
     def setup_once(self):

From pypy.commits at gmail.com  Wed Jun 15 09:01:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 15 Jun 2016 06:01:16 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: copy vector test from x86
 backend.
Message-ID: <5761519c.6372c20a.3456f.ffffbecd@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85177:9d282e6b23dd
Date: 2016-06-15 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/9d282e6b23dd/

Log:	copy vector test from x86 backend. some simplifications in the x86
	test (removed unused imports)

diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -42,13 +42,13 @@
 
     def setup(self):
         self.assembler = AssemblerPPC(self)
-        if detect_vsx():
-            self.vector_extension = True
-            # ??? self.vector_horizontal_operations = True
 
     @rgc.no_release_gil
     def setup_once(self):
         self.assembler.setup_once()
+        if detect_vsx():
+            self.vector_extension = True
+            # ??? self.vector_horizontal_operations = True
 
     @rgc.no_release_gil
     def finish_once(self):
diff --git a/rpython/jit/backend/ppc/test/test_ppcvector.py b/rpython/jit/backend/ppc/test/test_ppcvector.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/test/test_ppcvector.py
@@ -0,0 +1,22 @@
+import py
+from rpython.jit.backend.ppc.test import test_basic
+from rpython.jit.metainterp.test import test_vector
+
+
+class TestBasic(test_basic.JitPPCMixin, test_vector.VectorizeTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_basic.py
+    def setup_method(self, method):
+        clazz = self.CPUClass
+        def init(*args, **kwargs):
+            cpu = clazz(*args, **kwargs)
+            # > 95% can be executed, thus let's cheat here a little
+            cpu.supports_guard_gc_type = True
+            return cpu
+        self.CPUClass = init
+
+    def test_list_vectorize(self):
+        pass # needs support_guard_gc_type, disable for now
+
+    enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
+
diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -3,11 +3,7 @@
 from rpython.jit.backend.x86.test import test_basic
 from rpython.jit.backend.x86.test.test_assembler import \
         (TestRegallocPushPop as BaseTestAssembler)
-from rpython.jit.backend.detect_cpu import getcpuclass
-from rpython.jit.metainterp.history import ConstFloat
-from rpython.jit.metainterp.test import support, test_vector
-from rpython.jit.metainterp.warmspot import ll_meta_interp
-from rpython.rlib.jit import JitDriver
+from rpython.jit.metainterp.test import test_vector
 from rpython.rtyper.lltypesystem import lltype
 
 

From pypy.commits at gmail.com  Wed Jun 15 09:12:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 15 Jun 2016 06:12:55 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: test case skip (for cpu that
 does not support SIMD) changed a little
Message-ID: <57615457.8159c20a.cf0c1.19ba@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85178:376a87cb95fe
Date: 2016-06-15 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/376a87cb95fe/

Log:	test case skip (for cpu that does not support SIMD) changed a little
	now the tests are failing because of missing implementations of ppc

diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -7,7 +7,7 @@
 from rpython.jit.backend.ppc.arch import WORD
 from rpython.jit.backend.ppc.codebuilder import PPCBuilder
 from rpython.jit.backend.ppc import register as r
-
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
 
 class PPC_CPU(AbstractLLCPU):
 
diff --git a/rpython/jit/backend/ppc/test/test_ppcvector.py b/rpython/jit/backend/ppc/test/test_ppcvector.py
--- a/rpython/jit/backend/ppc/test/test_ppcvector.py
+++ b/rpython/jit/backend/ppc/test/test_ppcvector.py
@@ -1,6 +1,7 @@
 import py
 from rpython.jit.backend.ppc.test import test_basic
 from rpython.jit.metainterp.test import test_vector
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
 
 
 class TestBasic(test_basic.JitPPCMixin, test_vector.VectorizeTests):
@@ -15,6 +16,9 @@
             return cpu
         self.CPUClass = init
 
+    def supports_vector_ext(self):
+        return detect_vsx()
+
     def test_list_vectorize(self):
         pass # needs support_guard_gc_type, disable for now
 
diff --git a/rpython/jit/backend/x86/test/test_x86vector.py b/rpython/jit/backend/x86/test/test_x86vector.py
--- a/rpython/jit/backend/x86/test/test_x86vector.py
+++ b/rpython/jit/backend/x86/test/test_x86vector.py
@@ -6,7 +6,6 @@
 from rpython.jit.metainterp.test import test_vector
 from rpython.rtyper.lltypesystem import lltype
 
-
 class TestBasic(test_basic.Jit386Mixin, test_vector.VectorizeTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_basic.py
@@ -19,6 +18,9 @@
             return cpu
         self.CPUClass = init
 
+    def supports_vector_ext(self):
+        return self.CPUClass.vector_extension
+
     def test_list_vectorize(self):
         pass # needs support_guard_gc_type, disable for now
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -16,8 +16,6 @@
 from rpython.jit.backend.detect_cpu import getcpuclass
 
 CPU = getcpuclass()
-if not CPU.vector_extension:
-    py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
 @specialize.argtype(0,1)
 def malloc(T,n):
@@ -29,7 +27,8 @@
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
     def setup_method(self, method):
-        print "RUNNING", method.__name__
+        if not self.supports_vector_ext():
+            py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
     def meta_interp(self, f, args, policy=None, vec=True, vec_all=False):
         return ll_meta_interp(f, args, enable_opts=self.enable_opts,

From pypy.commits at gmail.com  Wed Jun 15 10:03:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 15 Jun 2016 07:03:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: a new vector register managed, 
Message-ID: <57616046.67c0c20a.bcb50.ffffc887@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85179:fcd54f26d4ff
Date: 2016-06-15 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/fcd54f26d4ff/

Log:	a new vector register managed, starting to impl. regalloc for loads

diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -29,6 +29,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.backend.ppc import callbuilder
 from rpython.rlib.rarithmetic import r_uint
+from rpython.jit.backend.ppc.vector_ext import VectorAssembler
 
 class IntOpAssembler(object):
         
@@ -1322,12 +1323,12 @@
         mc.b_abs(target)
         mc.copy_to_raw_memory(oldadr)
 
-
 class OpAssembler(IntOpAssembler, GuardOpAssembler,
                   MiscOpAssembler, FieldOpAssembler,
                   StrOpAssembler, CallOpAssembler,
                   UnicodeOpAssembler, ForceOpAssembler,
-                  AllocOpAssembler, FloatOpAssembler):
+                  AllocOpAssembler, FloatOpAssembler,
+                  VectorAssembler):
     _mixin_ = True
 
     def nop(self):
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -27,6 +27,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.rlib import rgc
 from rpython.rlib.rarithmetic import r_uint
+from rpython.jit.backend.ppc.vector_ext import VectorRegalloc
 
 LIMIT_LOOP_BREAK = 15000      # should be much smaller than 32 KB
 
@@ -135,6 +136,17 @@
         self.temp_boxes.append(box)
         return reg
 
+class VectorRegisterManager(RegisterManager):
+    all_regs              = r.MANAGED_VECTOR_REGS
+    box_types             = [INT, FLOAT]
+    save_around_call_regs = [] # ??? lookup the ABI
+    assert set(save_around_call_regs).issubset(all_regs)
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        RegisterManager.__init__(self, longevity, frame_manager, assembler)
+
+    def ensure_reg(self, box):
+        raise NotImplementedError
 
 class PPCFrameManager(FrameManager):
     def __init__(self, base_ofs):
@@ -155,8 +167,7 @@
         assert isinstance(loc, locations.StackLocation)
         return loc.position
 
-
-class Regalloc(BaseRegalloc):
+class Regalloc(BaseRegalloc, VectorRegalloc):
 
     def __init__(self, assembler=None):
         self.cpu = assembler.cpu
@@ -180,6 +191,8 @@
                                      assembler = self.assembler)
         self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm,
                                       assembler = self.assembler)
+        self.vrm = VectorRegisterManager(self.longevity, frame_manager = self.fm,
+                                      assembler = self.assembler)
         return operations
 
     def prepare_loop(self, inputargs, operations, looptoken, allgcrefs):
@@ -287,6 +300,7 @@
             self.assembler.mc.mark_op(op)
             self.rm.position = i
             self.fprm.position = i
+            self.vrm.position = i
             opnum = op.opnum
             if rop.has_no_side_effect(opnum) and op not in self.longevity:
                 i += 1
@@ -297,6 +311,8 @@
                 box = op.getarg(j)
                 if box.type != FLOAT:
                     self.rm.temp_boxes.append(box)
+                elif box.is_vector():
+                    self.vrm.temp_boxes.append(box)
                 else:
                     self.fprm.temp_boxes.append(box)
             #
@@ -309,6 +325,7 @@
             self.possibly_free_var(op)
             self.rm._check_invariants()
             self.fprm._check_invariants()
+            self.vrm._check_invariants()
             if self.assembler.mc.get_relative_pos() > self.limit_loop_break:
                 self.assembler.break_long_loop()
                 self.limit_loop_break = (self.assembler.mc.get_relative_pos() +
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -51,6 +51,8 @@
 
 MANAGED_FP_REGS = VOLATILES_FLOAT #+ NONVOLATILES_FLOAT
 
+MANAGED_VECTOR_REGS = ALL_VECTOR_REGS
+
 assert RCS1 in MANAGED_REGS and RCS1 in NONVOLATILES
 assert RCS2 in MANAGED_REGS and RCS2 in NONVOLATILES
 assert RCS3 in MANAGED_REGS and RCS3 in NONVOLATILES
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -17,7 +17,7 @@
         llop.debug_print(lltype.Void, msg)
     raise NotImplementedError(msg)
 
-class PPCVectorAssemblerMixin(object):
+class VectorAssembler(object):
     _mixin_ = True
 
     #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
@@ -510,30 +510,36 @@
     #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, resloc):
     #    self.mc.CVTPS2PD(resloc, arglocs[0])
 
-clas#s VectorRegallocMixin(object):
-    #_mixin_ = True
+class VectorRegalloc(object):
+    _mixin_ = True
 
-    #def _consider_vec_getarrayitem(self, op):
-    #    descr = op.getdescr()
-    #    assert isinstance(descr, ArrayDescr)
-    #    assert not descr.is_array_of_pointers() and \
-    #           not descr.is_array_of_structs()
-    #    itemsize, ofs, _ = unpack_arraydescr(descr)
-    #    integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-    #    aligned = False
-    #    args = op.getarglist()
-    #    base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-    #    ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
-    #    result_loc = self.force_allocate_reg(op)
-    #    self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
-    #                      imm(integer), imm(aligned)], result_loc)
+    def force_allocate_vector_reg(self, op):
+        forbidden_vars = self.vrm.temp_boxes
+        self.vrm.force_allocate_reg(op, forbidden_vars)
 
-    #consider_vec_getarrayitem_raw_i = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_raw_f = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_gc_i = _consider_vec_getarrayitem
-    #consider_vec_getarrayitem_gc_f = _consider_vec_getarrayitem
-    #consider_vec_raw_load_i = _consider_vec_getarrayitem
-    #consider_vec_raw_load_f = _consider_vec_getarrayitem
+    def _consider_load(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        aligned = False
+        args = op.getarglist()
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        base_loc = self.ensure_reg(a0)
+        ofs_loc = self.ensure_reg(a1)
+        result_loc = self.force_allocate_vector_reg(op)
+        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
+                          imm(integer), imm(aligned)], result_loc)
+
+    consider_vec_getarrayitem_raw_i = _consider_load
+    consider_vec_getarrayitem_raw_f = _consider_load
+    consider_vec_getarrayitem_gc_i = _consider_load
+    consider_vec_getarrayitem_gc_f = _consider_load
+    consider_vec_raw_load_i = _consider_load
+    consider_vec_raw_load_f = _consider_load
 
     #def _consider_vec_setarrayitem(self, op):
     #    descr = op.getdescr()

From pypy.commits at gmail.com  Wed Jun 15 13:14:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Jun 2016 10:14:58 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: first_created_object_uid()
Message-ID: <57618d12.e153c20a.8d0cb.ffff9c7f@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85180:91be90845d8d
Date: 2016-06-15 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/91be90845d8d/

Log:	first_created_object_uid()

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -71,6 +71,12 @@
     unique id greater or equal."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'u')
 
+def first_created_object_uid():
+    """Returns the creation number of the first object dynamically created
+    by the program.  Older objects are either prebuilt or created before
+    the first stop point."""
+    return llop.revdb_get_value(lltype.SignedLongLong, '1')
+
 @specialize.argtype(0)
 def get_unique_id(x):
     """Returns the creation number of the object 'x'.  For objects created
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -227,6 +227,7 @@
 static uint64_t total_stop_points;
 static uint64_t stopped_time;
 static uint64_t stopped_uid;
+static uint64_t first_created_uid;
 
 static void (*invoke_after_forward)(RPyString *);
 static RPyString *invoke_argument;
@@ -698,6 +699,8 @@
         fprintf(stderr, "stop_point_break overflow?\n");
         exit(1);
     }
+    if (frozen_num_pipes == 0)
+        first_created_uid = rpy_revdb.unique_id_seen;
 
     fprintf(stderr, "[%llu]",
             (unsigned long long)rpy_revdb.stop_point_seen);
@@ -877,6 +880,8 @@
         return rpy_revdb.stop_point_break;
     case 'u':       /* currently_created_objects() */
         return stopped_uid ? stopped_uid : rpy_revdb.unique_id_seen;
+    case '1':       /* first_created_object_uid() */
+        return first_created_uid;
     default:
         return -1;
     }
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -350,6 +350,9 @@
                 obj = revdb.get_tracked_object(Stuff)
                 revdb.send_output('None\n' if obj is None else
                                   ('obj.x=%d\n' % obj.x))
+            if cmdline == 'first-created-uid':
+                revdb.send_output('first-created-uid=%d\n' % (
+                    revdb.first_created_object_uid(),))
             revdb.send_output('blipped\n')
         lambda_blip = lambda: blip
         #
@@ -563,3 +566,22 @@
                       'obj.x=1001\r\n'
                       'blipped\r\n'
                       '(2)$ ')
+
+    def test_first_created_uid(self):
+        child = self.replay()
+        child.expectx('(3)$ ')
+        child.sendline('r first-created-uid')
+        child.expectx('<<>>\r\n')
+        child.expect('first-created-uid=(\d+)\r\n')
+        first_created_id = int(child.match.group(1))
+        child.expectx('blipped\r\n'
+                      '(3)$ ')
+        child.sendline('__go 1')
+        child.expectx('(1)$ ')
+        child.sendline('r print-id')
+        child.expect(re.escape('<<>>\r\n')
+                     + r'obj.x=1000 (\d+) (\d+)'
+                     + re.escape('\r\n'
+                                 'blipped\r\n'
+                                 '(1)$ '))
+        assert int(child.match.group(2)) == first_created_id

From pypy.commits at gmail.com  Wed Jun 15 14:02:30 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Wed, 15 Jun 2016 11:02:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix _call_has_no_star_args as starargs
 and kwargs are not existent anymore
Message-ID: <57619836.e1efc20a.3ad34.ffffb0a5@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85181:49e5e8c90e82
Date: 2016-06-15 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/49e5e8c90e82/

Log:	Fix _call_has_no_star_args as starargs and kwargs are not existent
	anymore

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1217,7 +1217,15 @@
                         call.args, call.keywords)
     
     def _call_has_no_star_args(self, call):
-        return not call.starargs and not call.kwargs
+        if call.args is not None:
+            for elt in call.args:
+                if isinstance(elt, ast.Starred):
+                    return False
+        if call.keywords is not None:
+            for kw in call.keywords:
+                if kw.arg is None:
+                    return False
+        return True
 
     def _call_has_simple_args(self, call):
         return self._call_has_no_star_args(call) and not call.keywords

From pypy.commits at gmail.com  Wed Jun 15 14:32:15 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 11:32:15 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: copy and start adapting code
 from rpython.translator.platform.distutils_platform
Message-ID: <57619f2f.56311c0a.1f5ae.fffff4d8@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85182:bd1fd28ffd7b
Date: 2016-06-14 22:08 +0100
http://bitbucket.org/pypy/pypy/changeset/bd1fd28ffd7b/

Log:	copy and start adapting code from
	rpython.translator.platform.distutils_platform

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/support.py
@@ -0,0 +1,133 @@
+import os
+import sys
+import py
+from rpython.translator.platform import log, CompilationError
+from rpython.translator.tool import stdoutcapture
+from rpython.translator.platform.distutils_platform import DistutilsPlatform
+
+rpy_platform = DistutilsPlatform()
+
+def log_spawned_cmd(spawn):
+    def spawn_and_log(cmd, *args, **kwds):
+        log.execute(' '.join(cmd))
+        return spawn(cmd, *args, **kwds)
+    return spawn_and_log
+
+CFLAGS = ['-O3']
+
+if os.name != 'nt':
+    so_ext = 'so'
+else:
+    so_ext = 'dll'
+
+def c_compile(cfilenames, eci, outputfilename=None, standalone=True):
+    self = rpy_platform
+    self._ensure_correct_math()
+    self.cfilenames = cfilenames
+    if standalone:
+        ext = ''
+    else:
+        ext = so_ext
+    self.standalone = standalone
+    self.libraries = list(eci.libraries)
+    self.include_dirs = list(eci.include_dirs)
+    self.library_dirs = list(eci.library_dirs)
+    self.compile_extra = list(eci.compile_extra)
+    self.link_extra = list(eci.link_extra)
+    self.frameworks = list(eci.frameworks)
+    if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
+        if 'm' not in self.libraries:
+            self.libraries.append('m')
+        self.compile_extra += CFLAGS + ['-fomit-frame-pointer']
+        if 'pthread' not in self.libraries:
+            self.libraries.append('pthread')
+        if self.name != 'sunos5':
+            self.compile_extra += ['-pthread']
+            self.link_extra += ['-pthread']
+        else:
+            self.compile_extra += ['-pthreads']
+            self.link_extra += ['-lpthread']
+    if self.name == 'win32':
+        self.link_extra += ['/DEBUG'] # generate .pdb file
+    if self.name == 'darwin':
+        # support Fink & Darwinports
+        for s in ('/sw/', '/opt/local/'):
+            if s + 'include' not in self.include_dirs and \
+                os.path.exists(s + 'include'):
+                self.include_dirs.append(s + 'include')
+            if s + 'lib' not in self.library_dirs and \
+                os.path.exists(s + 'lib'):
+                self.library_dirs.append(s + 'lib')
+        self.compile_extra += CFLAGS + ['-fomit-frame-pointer']
+        for framework in self.frameworks:
+            self.link_extra += ['-framework', framework]
+
+    if outputfilename is None:
+        self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
+    else:
+        self.outputfilename = py.path.local(outputfilename)
+    self.eci = eci
+    import distutils.errors
+    basename = self.outputfilename.new(ext='')
+    data = ''
+    try:
+        saved_environ = os.environ.copy()
+        c = stdoutcapture.Capture(mixed_out_err=True)
+        try:
+            self._build()
+        finally:
+            # workaround for a distutils bugs where some env vars can
+            # become longer and longer every time it is used
+            for key, value in saved_environ.items():
+                if os.environ.get(key) != value:
+                    os.environ[key] = value
+            foutput, foutput = c.done()
+            data = foutput.read()
+            if data:
+                fdump = basename.new(ext='errors').open("wb")
+                fdump.write(data)
+                fdump.close()
+    except (distutils.errors.CompileError,
+            distutils.errors.LinkError):
+        raise CompilationError('', data)
+    except:
+        print >>sys.stderr, data
+        raise
+    return self.outputfilename
+
+def _build(self):
+    from distutils.ccompiler import new_compiler
+    from distutils import sysconfig
+    compiler = new_compiler(force=1)
+    if self.cc is not None:
+        for c in '''compiler compiler_so compiler_cxx
+                    linker_exe linker_so'''.split():
+            compiler.executables[c][0] = self.cc
+    if not self.standalone:
+        sysconfig.customize_compiler(compiler) # XXX
+    compiler.spawn = log_spawned_cmd(compiler.spawn)
+    objects = []
+    for cfile in self.cfilenames:
+        cfile = py.path.local(cfile)
+        compile_extra = self.compile_extra[:]
+
+        old = cfile.dirpath().chdir()
+        try:
+            res = compiler.compile([cfile.basename],
+                                    include_dirs=self.eci.include_dirs,
+                                    extra_preargs=compile_extra)
+            assert len(res) == 1
+            cobjfile = py.path.local(res[0])
+            assert cobjfile.check()
+            objects.append(str(cobjfile))
+        finally:
+            old.chdir()
+
+    if self.standalone:
+        cmd = compiler.link_executable
+    else:
+        cmd = compiler.link_shared_object
+    cmd(objects, str(self.outputfilename),
+        libraries=self.eci.libraries,
+        extra_preargs=self.link_extra,
+        library_dirs=self.eci.library_dirs)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -8,7 +8,6 @@
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator.platform.distutils_platform import DistutilsPlatform
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -18,7 +17,7 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-rpy_platform = DistutilsPlatform()
+from .support import c_compile
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -52,7 +51,7 @@
         files = convert_sources_to_files(source_strings, dirname)
         source_files = files
     eci = ExternalCompilationInfo(include_dirs=include_dirs, **kwds)
-    soname = rpy_platform.compile(source_files, eci,
+    soname = c_compile(source_files, eci,
         outputfilename=str(dirname/modname),
         standalone=False)
     return soname

From pypy.commits at gmail.com  Wed Jun 15 14:32:18 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 11:32:18 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: remove random compiler flags
 that shouldn't matter for cpyext
Message-ID: <57619f32.06321c0a.527ad.ffffee3e@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85184:efcdfec61cef
Date: 2016-06-15 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/efcdfec61cef/

Log:	remove random compiler flags that shouldn't matter for cpyext

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -13,8 +13,6 @@
         return spawn(cmd, *args, **kwds)
     return spawn_and_log
 
-CFLAGS = ['-O3']
-
 if os.name != 'nt':
     so_ext = 'so'
 else:
@@ -22,7 +20,6 @@
 
 def c_compile(cfilenames, eci, outputfilename, standalone=True):
     self = rpy_platform
-    self._ensure_correct_math()
     self.cfilenames = cfilenames
     if standalone:
         ext = ''
@@ -38,15 +35,8 @@
     if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
         if 'm' not in self.libraries:
             self.libraries.append('m')
-        self.compile_extra += CFLAGS + ['-fomit-frame-pointer']
         if 'pthread' not in self.libraries:
             self.libraries.append('pthread')
-        if self.name != 'sunos5':
-            self.compile_extra += ['-pthread']
-            self.link_extra += ['-pthread']
-        else:
-            self.compile_extra += ['-pthreads']
-            self.link_extra += ['-lpthread']
     if self.name == 'win32':
         self.link_extra += ['/DEBUG'] # generate .pdb file
     if self.name == 'darwin':
@@ -58,7 +48,6 @@
             if s + 'lib' not in self.library_dirs and \
                 os.path.exists(s + 'lib'):
                 self.library_dirs.append(s + 'lib')
-        self.compile_extra += CFLAGS + ['-fomit-frame-pointer']
         for framework in self.frameworks:
             self.link_extra += ['-framework', framework]
 

From pypy.commits at gmail.com  Wed Jun 15 14:32:16 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 11:32:16 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: windows fix
Message-ID: <57619f30.815e1c0a.12db6.ffffac78@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85183:df4238f06f16
Date: 2016-06-15 17:10 +0100
http://bitbucket.org/pypy/pypy/changeset/df4238f06f16/

Log:	windows fix

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -20,7 +20,7 @@
 else:
     so_ext = 'dll'
 
-def c_compile(cfilenames, eci, outputfilename=None, standalone=True):
+def c_compile(cfilenames, eci, outputfilename, standalone=True):
     self = rpy_platform
     self._ensure_correct_math()
     self.cfilenames = cfilenames
@@ -62,10 +62,7 @@
         for framework in self.frameworks:
             self.link_extra += ['-framework', framework]
 
-    if outputfilename is None:
-        self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-    else:
-        self.outputfilename = py.path.local(outputfilename)
+    self.outputfilename = py.path.local(outputfilename).new(ext=ext)
     self.eci = eci
     import distutils.errors
     basename = self.outputfilename.new(ext='')

From pypy.commits at gmail.com  Wed Jun 15 19:14:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 16:14:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix -A segfault
Message-ID: <5761e159.2457c20a.ec0a.05f6@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r85185:54fc5ca89502
Date: 2016-06-16 00:13 +0100
http://bitbucket.org/pypy/pypy/changeset/54fc5ca89502/

Log:	Fix -A segfault

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -43,15 +43,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -195,7 +186,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -463,9 +454,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -508,12 +499,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/

From pypy.commits at gmail.com  Wed Jun 15 19:17:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 16:17:54 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: kill standalone option (always
 False)
Message-ID: <5761e222.a758c20a.2867f.2d15@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85186:13b38ae4c4be
Date: 2016-06-15 19:51 +0100
http://bitbucket.org/pypy/pypy/changeset/13b38ae4c4be/

Log:	kill standalone option (always False)

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -18,14 +18,10 @@
 else:
     so_ext = 'dll'
 
-def c_compile(cfilenames, eci, outputfilename, standalone=True):
+def c_compile(cfilenames, eci, outputfilename):
     self = rpy_platform
     self.cfilenames = cfilenames
-    if standalone:
-        ext = ''
-    else:
-        ext = so_ext
-    self.standalone = standalone
+    ext = so_ext
     self.libraries = list(eci.libraries)
     self.include_dirs = list(eci.include_dirs)
     self.library_dirs = list(eci.library_dirs)
@@ -60,7 +56,7 @@
         saved_environ = os.environ.copy()
         c = stdoutcapture.Capture(mixed_out_err=True)
         try:
-            self._build()
+            _build(self)
         finally:
             # workaround for a distutils bugs where some env vars can
             # become longer and longer every time it is used
@@ -85,12 +81,7 @@
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
     compiler = new_compiler(force=1)
-    if self.cc is not None:
-        for c in '''compiler compiler_so compiler_cxx
-                    linker_exe linker_so'''.split():
-            compiler.executables[c][0] = self.cc
-    if not self.standalone:
-        sysconfig.customize_compiler(compiler) # XXX
+    sysconfig.customize_compiler(compiler) # XXX
     compiler.spawn = log_spawned_cmd(compiler.spawn)
     objects = []
     for cfile in self.cfilenames:
@@ -109,11 +100,7 @@
         finally:
             old.chdir()
 
-    if self.standalone:
-        cmd = compiler.link_executable
-    else:
-        cmd = compiler.link_shared_object
-    cmd(objects, str(self.outputfilename),
+    compiler.link_shared_object(objects, str(self.outputfilename),
         libraries=self.eci.libraries,
         extra_preargs=self.link_extra,
         library_dirs=self.eci.library_dirs)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -51,9 +51,7 @@
         files = convert_sources_to_files(source_strings, dirname)
         source_files = files
     eci = ExternalCompilationInfo(include_dirs=include_dirs, **kwds)
-    soname = c_compile(source_files, eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = c_compile(source_files, eci, outputfilename=str(dirname/modname))
     return soname
 
 def compile_extension_module(space, modname, include_dirs=[],

From pypy.commits at gmail.com  Wed Jun 15 19:17:56 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 15 Jun 2016 16:17:56 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Fix -A segfault
Message-ID: <5761e224.430ac20a.33013.1208@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85187:334372c13d02
Date: 2016-06-16 00:13 +0100
http://bitbucket.org/pypy/pypy/changeset/334372c13d02/

Log:	Fix -A segfault

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -43,15 +43,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -195,7 +186,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -463,9 +454,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -508,12 +499,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/

From pypy.commits at gmail.com  Thu Jun 16 05:48:54 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 16 Jun 2016 02:48:54 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: seems there is still a case
 where a condition with a non-constant is made
Message-ID: <57627606.4a2e1c0a.79e06.fffff4ab@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: guard-compatible
Changeset: r85188:a48045cfa46a
Date: 2016-06-15 14:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a48045cfa46a/

Log:	seems there is still a case where a condition with a non-constant is
	made

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -58,11 +58,14 @@
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
         # replace further arguments by constants, if the optimizer knows them
         # already
+        last_nonconst_index = -1
         for i in range(2, op.numargs()):
             arg = op.getarg(i)
             constarg = optimizer.get_constant_box(arg)
             if constarg is not None:
                 op.setarg(i, constarg)
+            else:
+                last_nonconst_index = i
         copied_op = op.copy()
         copied_op.setarg(1, self.known_valid)
         if op.numargs() == 2:
@@ -70,7 +73,11 @@
         arg2 = copied_op.getarg(2)
         if arg2.is_constant():
             # already a constant, can just use PureCallCondition
+            if last_nonconst_index != -1:
+                return None, None # a non-constant argument, can't optimize
             return copied_op, PureCallCondition(op, optimizer)
+        if last_nonconst_index != 2:
+            return None, None
 
         # really simple-minded pattern matching
         # the order of things is like this:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py
@@ -157,6 +157,34 @@
         assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.myptr))
         assert len(descr._compatibility_conditions.conditions) == 2
 
+    def test_guard_compatible_call_pure_not_const(self):
+        call_pure_results = {
+            (ConstInt(123), ConstPtr(self.myptr), ConstInt(5), ConstInt(5)): ConstInt(5),
+            (ConstInt(124), ConstPtr(self.myptr), ConstInt(5), ConstInt(5)): ConstInt(7),
+        }
+        ops = """
+        [p1, i2]
+        pvirtual = new_with_vtable(descr=nodesize)
+        setfield_gc(pvirtual, 5, descr=valuedescr)
+        i1 = getfield_gc_i(pvirtual, descr=valuedescr)
+        guard_compatible(p1, ConstPtr(myptr)) []
+        i3 = call_pure_i(123, p1, i1, i2, descr=plaincalldescr)
+        escape_n(i3)
+        i5 = call_pure_i(124, p1, i1, i2, descr=plaincalldescr)
+        escape_n(i5)
+        jump(ConstPtr(myptr), 5)
+        """
+        expected = """
+        [p1, i2]
+        guard_compatible(p1, ConstPtr(myptr)) []
+        i3 = call_i(123, p1, 5, i2, descr=plaincalldescr)
+        escape_n(i3)
+        i5 = call_i(124, p1, 5, i2, descr=plaincalldescr)
+        escape_n(i5)
+        jump(ConstPtr(myptr), 5)
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+
     def test_deduplicate_conditions(self):
         call_pure_results = {
             (ConstInt(123), ConstPtr(self.myptr)): ConstInt(5),
@@ -222,3 +250,31 @@
         assert descr._compatibility_conditions is not None
         assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.quasiptr))
         assert len(descr._compatibility_conditions.conditions) == 1
+
+    def test_quasiimmut_nonconst(self):
+        ops = """
+        [p1, i5]
+        guard_compatible(p1, ConstPtr(quasiptr)) []
+        quasiimmut_field(p1, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        i0 = getfield_gc_i(p1, descr=quasifielddescr)
+        i1 = call_pure_i(123, p1, i0, i5, descr=nonwritedescr)
+        i4 = call_pure_i(123, p1, i0, i5, descr=nonwritedescr)
+        escape_n(i1)
+        escape_n(i4)
+        jump(p1, i5)
+        """
+        expected = """
+        [p1, i5]
+        guard_compatible(p1, ConstPtr(quasiptr)) []
+        guard_not_invalidated() []
+        i0 = getfield_gc_i(p1, descr=quasifielddescr) # will be removed by the backend
+        i1 = call_i(123, p1, i0, i5, descr=nonwritedescr)
+        escape_n(i1)
+        escape_n(i1)
+        jump(p1, i5)
+        """
+        call_pure_results = {
+            (ConstInt(123), ConstPtr(self.quasiptr), ConstInt(-4247)): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)

From pypy.commits at gmail.com  Thu Jun 16 08:32:26 2016
From: pypy.commits at gmail.com (alex_gaynor)
Date: Thu, 16 Jun 2016 05:32:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Bumped greenlet version for upstream.
Message-ID: <57629c5a.c255c20a.588fa.5c54@mx.google.com>

Author: Alex Gaynor 
Branch: 
Changeset: r85189:6fcdbf16a5f9
Date: 2016-06-16 08:31 -0400
http://bitbucket.org/pypy/pypy/changeset/6fcdbf16a5f9/

Log:	Bumped greenlet version for upstream.

	There are no changes that we need to port over.

diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions

From pypy.commits at gmail.com  Thu Jun 16 12:02:45 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 16 Jun 2016 09:02:45 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added incomplete impl. for
 vec_raw_load_i
Message-ID: <5762cda5.e457c20a.47e40.ffffbebf@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85190:517eb8e476c8
Date: 2016-06-16 15:37 +0200
http://bitbucket.org/pypy/pypy/changeset/517eb8e476c8/

Log:	added incomplete impl. for vec_raw_load_i

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -10,6 +10,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.backend.ppc.locations import imm
 
 def not_implemented(msg):
     msg = '[ppc/vector_ext] %s\n' % msg
@@ -20,6 +21,48 @@
 class VectorAssembler(object):
     _mixin_ = True
 
+    def _emit_getitem(self, op, arglocs, regalloc):
+        # prepares item scale (raw_load does not)
+        resloc, base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        scale = get_scale(size_loc.value)
+        xxx
+        src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
+        self._vec_load(resloc, src_addr, integer_loc.value,
+                       size_loc.value, aligned_loc.value)
+    
+    emit_vec_getarrayitem_raw_i = _emit_getitem
+    emit_vec_getarrayitem_raw_f = _emit_getitem
+
+    emit_vec_getarrayitem_gc_i = _emit_getitem
+    emit_vec_getarrayitem_gc_f = _emit_getitem
+
+    def _emit_load(self, op, arglocs, regalloc):
+        resloc, base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        #src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
+        assert ofs.value == 0
+        self._vec_load(resloc, base_loc, ofs_loc, integer_loc.value,
+                       size_loc.value, aligned_loc.value)
+
+    emit_vec_raw_load_i = _emit_load
+    emit_vec_raw_load_f = _emit_load
+
+    def _vec_load(self, resloc, baseloc, indexloc, integer, itemsize, aligned):
+        if integer:
+            if itemsize == 4:
+                self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+            elif itemsize == 8:
+                self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
+            else:
+                raise NotImplementedError
+        else:
+            if itemsize == 4:
+                self.mc.MOVUPS(resloc, src_addr)
+            elif itemsize == 8:
+                self.mc.MOVUPD(resloc, src_addr)
+            else:
+                raise NotImplementedError
+
+
     #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
     #    self.implement_guard(guard_token)
 
@@ -124,43 +167,8 @@
 
     #    not_implemented("reduce sum for %s not impl." % arg)
 
-    #def _genop_vec_getarrayitem(self, op, arglocs, resloc):
-    #    # considers item scale (raw_load does not)
-    #    base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-    #    scale = get_scale(size_loc.value)
-    #    src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale)
-    #    self._vec_load(resloc, src_addr, integer_loc.value,
-    #                   size_loc.value, aligned_loc.value)
-    #
-    #genop_vec_getarrayitem_raw_i = _genop_vec_getarrayitem
-    #genop_vec_getarrayitem_raw_f = _genop_vec_getarrayitem
-    #
-    #genop_vec_getarrayitem_gc_i = _genop_vec_getarrayitem
-    #genop_vec_getarrayitem_gc_f = _genop_vec_getarrayitem
-
-    #def _genop_vec_raw_load(self, op, arglocs, resloc):
-    #    base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
-    #    src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
-    #    self._vec_load(resloc, src_addr, integer_loc.value,
-    #                   size_loc.value, aligned_loc.value)
-
-    #genop_vec_raw_load_i = _genop_vec_raw_load
-    #genop_vec_raw_load_f = _genop_vec_raw_load
-
-    #def _vec_load(self, resloc, src_addr, integer, itemsize, aligned):
-    #    if integer:
-    #        if aligned:
-    #            self.mc.MOVDQA(resloc, src_addr)
-    #        else:
-    #            self.mc.MOVDQU(resloc, src_addr)
-    #    else:
-    #        if itemsize == 4:
-    #            self.mc.MOVUPS(resloc, src_addr)
-    #        elif itemsize == 8:
-    #            self.mc.MOVUPD(resloc, src_addr)
-
     #def _genop_discard_vec_setarrayitem(self, op, arglocs):
-    #    # considers item scale (raw_store does not)
+    #    # prepares item scale (raw_store does not)
     #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
     #    scale = get_scale(size_loc.value)
     #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
@@ -515,9 +523,9 @@
 
     def force_allocate_vector_reg(self, op):
         forbidden_vars = self.vrm.temp_boxes
-        self.vrm.force_allocate_reg(op, forbidden_vars)
+        return self.vrm.force_allocate_reg(op, forbidden_vars)
 
-    def _consider_load(self, op):
+    def _prepare_load(self, op):
         descr = op.getdescr()
         assert isinstance(descr, ArrayDescr)
         assert not descr.is_array_of_pointers() and \
@@ -531,17 +539,17 @@
         base_loc = self.ensure_reg(a0)
         ofs_loc = self.ensure_reg(a1)
         result_loc = self.force_allocate_vector_reg(op)
-        self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
-                          imm(integer), imm(aligned)], result_loc)
+        return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
+                imm(integer), imm(aligned)]
 
-    consider_vec_getarrayitem_raw_i = _consider_load
-    consider_vec_getarrayitem_raw_f = _consider_load
-    consider_vec_getarrayitem_gc_i = _consider_load
-    consider_vec_getarrayitem_gc_f = _consider_load
-    consider_vec_raw_load_i = _consider_load
-    consider_vec_raw_load_f = _consider_load
+    prepare_vec_getarrayitem_raw_i = _prepare_load
+    prepare_vec_getarrayitem_raw_f = _prepare_load
+    prepare_vec_getarrayitem_gc_i = _prepare_load
+    prepare_vec_getarrayitem_gc_f = _prepare_load
+    prepare_vec_raw_load_i = _prepare_load
+    prepare_vec_raw_load_f = _prepare_load
 
-    #def _consider_vec_setarrayitem(self, op):
+    #def _prepare_vec_setarrayitem(self, op):
     #    descr = op.getdescr()
     #    assert isinstance(descr, ArrayDescr)
     #    assert not descr.is_array_of_pointers() and \
@@ -557,11 +565,11 @@
     #    self.perform_discard(op, [base_loc, ofs_loc, value_loc,
     #                             imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
 
-    #consider_vec_setarrayitem_raw = _consider_vec_setarrayitem
-    #consider_vec_setarrayitem_gc = _consider_vec_setarrayitem
-    #consider_vec_raw_store = _consider_vec_setarrayitem
+    #prepare_vec_setarrayitem_raw = _prepare_vec_setarrayitem
+    #prepare_vec_setarrayitem_gc = _prepare_vec_setarrayitem
+    #prepare_vec_raw_store = _prepare_vec_setarrayitem
 
-    #def consider_vec_arith(self, op):
+    #def prepare_vec_arith(self, op):
     #    lhs = op.getarg(0)
     #    assert isinstance(op, VectorOp)
     #    size = op.bytesize
@@ -570,27 +578,27 @@
     #    loc0 = self.xrm.force_result_in_reg(op, op.getarg(0), args)
     #    self.perform(op, [loc0, loc1, imm(size)], loc0)
 
-    #consider_vec_int_add = consider_vec_arith
-    #consider_vec_int_sub = consider_vec_arith
-    #consider_vec_int_mul = consider_vec_arith
-    #consider_vec_float_add = consider_vec_arith
-    #consider_vec_float_sub = consider_vec_arith
-    #consider_vec_float_mul = consider_vec_arith
-    #consider_vec_float_truediv = consider_vec_arith
-    #del consider_vec_arith
+    #prepare_vec_int_add = prepare_vec_arith
+    #prepare_vec_int_sub = prepare_vec_arith
+    #prepare_vec_int_mul = prepare_vec_arith
+    #prepare_vec_float_add = prepare_vec_arith
+    #prepare_vec_float_sub = prepare_vec_arith
+    #prepare_vec_float_mul = prepare_vec_arith
+    #prepare_vec_float_truediv = prepare_vec_arith
+    #del prepare_vec_arith
 
-    #def consider_vec_arith_unary(self, op):
+    #def prepare_vec_arith_unary(self, op):
     #    lhs = op.getarg(0)
     #    assert isinstance(lhs, VectorOp)
     #    args = op.getarglist()
     #    res = self.xrm.force_result_in_reg(op, op.getarg(0), args)
     #    self.perform(op, [res, imm(lhs.bytesize)], res)
 
-    #consider_vec_float_neg = consider_vec_arith_unary
-    #consider_vec_float_abs = consider_vec_arith_unary
-    #del consider_vec_arith_unary
+    #prepare_vec_float_neg = prepare_vec_arith_unary
+    #prepare_vec_float_abs = prepare_vec_arith_unary
+    #del prepare_vec_arith_unary
 
-    #def consider_vec_logic(self, op):
+    #def prepare_vec_logic(self, op):
     #    lhs = op.getarg(0)
     #    assert isinstance(lhs, VectorOp)
     #    args = op.getarglist()
@@ -598,7 +606,7 @@
     #    result = self.xrm.force_result_in_reg(op, op.getarg(0), args)
     #    self.perform(op, [source, imm(lhs.bytesize)], result)
 
-    #def consider_vec_float_eq(self, op):
+    #def prepare_vec_float_eq(self, op):
     #    assert isinstance(op, VectorOp)
     #    lhs = op.getarg(0)
     #    assert isinstance(lhs, VectorOp)
@@ -607,16 +615,16 @@
     #    lhsloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
     #    self.perform(op, [lhsloc, rhsloc, imm(lhs.bytesize)], lhsloc)
 
-    #consider_vec_float_ne = consider_vec_float_eq
-    #consider_vec_int_eq = consider_vec_float_eq
-    #consider_vec_int_ne = consider_vec_float_eq
+    #prepare_vec_float_ne = prepare_vec_float_eq
+    #prepare_vec_int_eq = prepare_vec_float_eq
+    #prepare_vec_int_ne = prepare_vec_float_eq
 
-    #consider_vec_int_and = consider_vec_logic
-    #consider_vec_int_or = consider_vec_logic
-    #consider_vec_int_xor = consider_vec_logic
-    #del consider_vec_logic
+    #prepare_vec_int_and = prepare_vec_logic
+    #prepare_vec_int_or = prepare_vec_logic
+    #prepare_vec_int_xor = prepare_vec_logic
+    #del prepare_vec_logic
 
-    #def consider_vec_pack_i(self, op):
+    #def prepare_vec_pack_i(self, op):
     #    # new_res = vec_pack_i(res, src, index, count)
     #    assert isinstance(op, VectorOp)
     #    arg = op.getarg(1)
@@ -633,9 +641,9 @@
     #               imm(count.value), imm(op.bytesize)]
     #    self.perform(op, arglocs, resloc)
 
-    #consider_vec_pack_f = consider_vec_pack_i
+    #prepare_vec_pack_f = prepare_vec_pack_i
 
-    #def consider_vec_unpack_i(self, op):
+    #def prepare_vec_unpack_i(self, op):
     #    assert isinstance(op, VectorOp)
     #    index = op.getarg(1)
     #    count = op.getarg(2)
@@ -657,9 +665,9 @@
     #    arglocs = [resloc, srcloc, imm(residx), imm(index.value), imm(count.value), imm(size)]
     #    self.perform(op, arglocs, resloc)
 
-    #consider_vec_unpack_f = consider_vec_unpack_i
+    #prepare_vec_unpack_f = prepare_vec_unpack_i
 
-    #def consider_vec_expand_f(self, op):
+    #def prepare_vec_expand_f(self, op):
     #    assert isinstance(op, VectorOp)
     #    arg = op.getarg(0)
     #    args = op.getarglist()
@@ -671,7 +679,7 @@
     #        srcloc = resloc
     #    self.perform(op, [srcloc, imm(op.bytesize)], resloc)
 
-    #def consider_vec_expand_i(self, op):
+    #def prepare_vec_expand_i(self, op):
     #    assert isinstance(op, VectorOp)
     #    arg = op.getarg(0)
     #    args = op.getarglist()
@@ -682,7 +690,7 @@
     #    resloc = self.xrm.force_allocate_reg(op, args)
     #    self.perform(op, [srcloc, imm(op.bytesize)], resloc)
 
-    #def consider_vec_int_signext(self, op):
+    #def prepare_vec_int_signext(self, op):
     #    assert isinstance(op, VectorOp)
     #    args = op.getarglist()
     #    resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
@@ -692,7 +700,7 @@
     #    assert size > 0
     #    self.perform(op, [resloc, imm(size), imm(op.bytesize)], resloc)
 
-    #def consider_vec_int_is_true(self, op):
+    #def prepare_vec_int_is_true(self, op):
     #    args = op.getarglist()
     #    arg = op.getarg(0)
     #    assert isinstance(arg, VectorOp)
@@ -700,30 +708,30 @@
     #    resloc = self.xrm.force_result_in_reg(op, arg, args)
     #    self.perform(op, [resloc,imm(arg.bytesize)], None)
 
-    #def _consider_vec(self, op):
+    #def _prepare_vec(self, op):
     #    # pseudo instruction, needed to create a new variable
     #    self.xrm.force_allocate_reg(op)
 
-    #consider_vec_i = _consider_vec
-    #consider_vec_f = _consider_vec
+    #prepare_vec_i = _prepare_vec
+    #prepare_vec_f = _prepare_vec
 
-    #def consider_vec_cast_float_to_int(self, op):
+    #def prepare_vec_cast_float_to_int(self, op):
     #    args = op.getarglist()
     #    srcloc = self.make_sure_var_in_reg(op.getarg(0), args)
     #    resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args)
     #    self.perform(op, [srcloc], resloc)
 
-    #consider_vec_cast_int_to_float = consider_vec_cast_float_to_int
-    #consider_vec_cast_float_to_singlefloat = consider_vec_cast_float_to_int
-    #consider_vec_cast_singlefloat_to_float = consider_vec_cast_float_to_int
+    #prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int
+    #prepare_vec_cast_float_to_singlefloat = prepare_vec_cast_float_to_int
+    #prepare_vec_cast_singlefloat_to_float = prepare_vec_cast_float_to_int
 
-    #def consider_vec_guard_true(self, op):
+    #def prepare_vec_guard_true(self, op):
     #    arg = op.getarg(0)
     #    loc = self.loc(arg)
     #    self.assembler.guard_vector(op, self.loc(arg), True)
     #    self.perform_guard(op, [], None)
 
-    #def consider_vec_guard_false(self, op):
+    #def prepare_vec_guard_false(self, op):
     #    arg = op.getarg(0)
     #    loc = self.loc(arg)
     #    self.assembler.guard_vector(op, self.loc(arg), False)

From pypy.commits at gmail.com  Thu Jun 16 12:02:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 16 Jun 2016 09:02:47 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added vector add for powerpc,
 test modified to use hypothesis
Message-ID: <5762cda7.871a1c0a.d9232.ffff8bee@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85191:daa1de3f481f
Date: 2016-06-16 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/daa1de3f481f/

Log:	added vector add for powerpc, test modified to use hypothesis

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -62,6 +62,7 @@
 XFX = Form("CRM", "rS", "XO1")
 XLL = Form("LL", "XO1")
 XX1 = Form("vrT", "rA", "rB", "XO1")
+VX = Form("lvrT", "lvrA", "lvrB", "XO8")
 
 MI = Form("rA", "rS", "SH", "MB", "ME", "Rc")
 MB = Form("rA", "rS", "rB", "MB", "ME", "Rc")
@@ -584,6 +585,9 @@
     stxvd2x = XX1(31, XO1=972)
     stxvw4x = XX1(31, XO1=908)
 
+    # integer
+    vaddudm = VX(4, XO8=192)
+
 class PPCAssembler(BasicPPCAssembler, PPCVSXAssembler):
     BA = BasicPPCAssembler
 
diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py
--- a/rpython/jit/backend/ppc/ppc_field.py
+++ b/rpython/jit/backend/ppc/ppc_field.py
@@ -44,6 +44,13 @@
     "TO":     ( 6, 10),
     "UIMM":   (16, 31),
     "vrT":    (6,  31, 'unsigned', regname._V, 'overlap'),
+    # low vector register T (low in a sense:
+    # can only address 32 vector registers)
+    "lvrT":   (6,  10, 'unsigned', regname._V),
+    # low vector register A
+    "lvrA":   (11, 15, 'unsigned', regname._V),
+    # low vector register B
+    "lvrB":   (16, 20, 'unsigned', regname._V),
     "XO1":    (21, 30),
     "XO2":    (22, 30),
     "XO3":    (26, 30),
@@ -51,6 +58,7 @@
     "XO5":    (27, 29),
     "XO6":    (21, 29),
     "XO7":    (27, 30),
+    "XO8":    (21, 31),
     "LL":     ( 9, 10),
 }
 
@@ -102,16 +110,16 @@
         value = super(sh, self).decode(inst)
         return (value & 32) << 5 | (value >> 10 & 31)
 
-class tx(Field):
-    def encode(self, value):
-        value = (value & 31) << 20 | (value & 32) >> 5
-        return super(tx, self).encode(value)
-    def decode(self, inst):
-        value = super(tx, self).decode(inst)
-        return (value & 32) << 5 | (value >> 20 & 31)
-    def r(self):
-        import pdb; pdb.set_trace()
-        return super(tx, self).r()
+# ??? class tx(Field):
+# ???     def encode(self, value):
+# ???         value = (value & 31) << 20 | (value & 32) >> 5
+# ???         return super(tx, self).encode(value)
+# ???     def decode(self, inst):
+# ???         value = super(tx, self).decode(inst)
+# ???         return (value & 32) << 5 | (value >> 20 & 31)
+# ???     def r(self):
+# ???         import pdb; pdb.set_trace()
+# ???         return super(tx, self).r()
 # other special fields?
 
 ppc_fields = {
@@ -121,7 +129,7 @@
     "mbe": mbe("mbe",   *fields["mbe"]),
     "sh":  sh("sh",     *fields["sh"]),
     "spr": spr("spr",   *fields["spr"]),
-    "vrT": tx("vrT",    *fields["vrT"]),
+    # ??? "vrT": tx("vrT",    *fields["vrT"]),
 }
 
 for f in fields:
diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py
--- a/rpython/jit/backend/ppc/rassemblermaker.py
+++ b/rpython/jit/backend/ppc/rassemblermaker.py
@@ -47,7 +47,7 @@
             body.append('sh1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value))
             value = 'sh1'
         elif field.name == 'vrT':
-            body.append('vrT1 = (%s & 31) << 20 | (%s & 32) >> 5' % (value, value))
+            body.append('vrT1 = (%s & 31) << 21 | (%s & 32) >> 5' % (value, value))
             value = 'vrT1'
         if isinstance(field, IField):
             body.append('v |= ((%3s >> 2) & r_uint(%#05x)) << 2' % (value, field.mask))
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -256,6 +256,8 @@
         if var is not None:
             if var.type == FLOAT:
                 self.fprm.possibly_free_var(var)
+            elif var.is_vector() and var.type != VOID:
+                self.vrm.possibly_free_var(var)
             else:
                 self.rm.possibly_free_var(var)
 
@@ -309,10 +311,10 @@
             #
             for j in range(op.numargs()):
                 box = op.getarg(j)
-                if box.type != FLOAT:
+                if box.is_vector():
+                    self.vrm.temp_boxes.append(box)
+                elif box.type != FLOAT:
                     self.rm.temp_boxes.append(box)
-                elif box.is_vector():
-                    self.vrm.temp_boxes.append(box)
                 else:
                     self.fprm.temp_boxes.append(box)
             #
@@ -436,6 +438,7 @@
         # temporary boxes and all the current operation's arguments
         self.rm.free_temp_vars()
         self.fprm.free_temp_vars()
+        self.vrm.free_temp_vars()
 
     # ******************************************************
     # *         P R E P A R E  O P E R A T I O N S         * 
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -49,9 +49,9 @@
     def _vec_load(self, resloc, baseloc, indexloc, integer, itemsize, aligned):
         if integer:
             if itemsize == 4:
+                self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
+            elif itemsize == 8:
                 self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
-            elif itemsize == 8:
-                self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
             else:
                 raise NotImplementedError
         else:
@@ -62,6 +62,48 @@
             else:
                 raise NotImplementedError
 
+    def _emit_vec_setitem(self, op, arglocs, regalloc):
+        # prepares item scale (raw_store does not)
+        base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
+        scale = get_scale(size_loc.value)
+        dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
+        self._vec_store(dest_loc, value_loc, integer_loc.value,
+                        size_loc.value, aligned_loc.value)
+
+    genop_discard_vec_setarrayitem_raw = _emit_vec_setitem
+    genop_discard_vec_setarrayitem_gc = _emit_vec_setitem
+
+    def emit_vec_raw_store(self, op, arglocs, regalloc):
+        baseloc, ofsloc, valueloc, size_loc, baseofs, \
+            integer_loc, aligned_loc = arglocs
+        #dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
+        assert baseofs.value == 0
+        self._vec_store(baseloc, ofsloc, valueloc, integer_loc.value,
+                        size_loc.value, aligned_loc.value)
+
+    def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, aligned):
+        if integer:
+            if itemsize == 4:
+                self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
+            elif itemsize == 8:
+                self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+            else:
+                raise NotImplementedError
+        else:
+            raise NotImplementedError
+
+
+    def emit_vec_int_add(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            raise NotImplementedError
+        elif size == 2:
+            raise NotImplementedError
+        elif size == 4:
+            raise NotImplementedError
+        elif size == 8:
+            self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
 
     #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
     #    self.implement_guard(guard_token)
@@ -167,35 +209,6 @@
 
     #    not_implemented("reduce sum for %s not impl." % arg)
 
-    #def _genop_discard_vec_setarrayitem(self, op, arglocs):
-    #    # prepares item scale (raw_store does not)
-    #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-    #    scale = get_scale(size_loc.value)
-    #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, scale)
-    #    self._vec_store(dest_loc, value_loc, integer_loc.value,
-    #                    size_loc.value, aligned_loc.value)
-
-    #genop_discard_vec_setarrayitem_raw = _genop_discard_vec_setarrayitem
-    #genop_discard_vec_setarrayitem_gc = _genop_discard_vec_setarrayitem
-
-    #def genop_discard_vec_raw_store(self, op, arglocs):
-    #    base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
-    #    dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
-    #    self._vec_store(dest_loc, value_loc, integer_loc.value,
-    #                    size_loc.value, aligned_loc.value)
-
-    #def _vec_store(self, dest_loc, value_loc, integer, itemsize, aligned):
-    #    if integer:
-    #        if aligned:
-    #            self.mc.MOVDQA(dest_loc, value_loc)
-    #        else:
-    #            self.mc.MOVDQU(dest_loc, value_loc)
-    #    else:
-    #        if itemsize == 4:
-    #            self.mc.MOVUPS(dest_loc, value_loc)
-    #        elif itemsize == 8:
-    #            self.mc.MOVUPD(dest_loc, value_loc)
-
     #def genop_vec_int_is_true(self, op, arglocs, resloc):
     #    loc, sizeloc = arglocs
     #    temp = X86_64_XMM_SCRATCH_REG
@@ -219,18 +232,6 @@
     #        # There is no 64x64 bit packed mul. For 8 bit either. It is questionable if it gives any benefit?
     #        not_implemented("int8/64 mul")
 
-    #def genop_vec_int_add(self, op, arglocs, resloc):
-    #    loc0, loc1, size_loc = arglocs
-    #    size = size_loc.value
-    #    if size == 1:
-    #        self.mc.PADDB(loc0, loc1)
-    #    elif size == 2:
-    #        self.mc.PADDW(loc0, loc1)
-    #    elif size == 4:
-    #        self.mc.PADDD(loc0, loc1)
-    #    elif size == 8:
-    #        self.mc.PADDQ(loc0, loc1)
-
     #def genop_vec_int_sub(self, op, arglocs, resloc):
     #    loc0, loc1, size_loc = arglocs
     #    size = size_loc.value
@@ -525,6 +526,11 @@
         forbidden_vars = self.vrm.temp_boxes
         return self.vrm.force_allocate_reg(op, forbidden_vars)
 
+    def ensure_vector_reg(self, box):
+        loc = self.vrm.make_sure_var_in_reg(box,
+                           forbidden_vars=self.vrm.temp_boxes)
+        return loc
+
     def _prepare_load(self, op):
         descr = op.getdescr()
         assert isinstance(descr, ArrayDescr)
@@ -549,43 +555,49 @@
     prepare_vec_raw_load_i = _prepare_load
     prepare_vec_raw_load_f = _prepare_load
 
-    #def _prepare_vec_setarrayitem(self, op):
-    #    descr = op.getdescr()
-    #    assert isinstance(descr, ArrayDescr)
-    #    assert not descr.is_array_of_pointers() and \
-    #           not descr.is_array_of_structs()
-    #    itemsize, ofs, _ = unpack_arraydescr(descr)
-    #    args = op.getarglist()
-    #    base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-    #    value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
-    #    ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+    def prepare_vec_arith(self, op):
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        assert isinstance(op, VectorOp)
+        size = op.bytesize
+        args = op.getarglist()
+        loc0 = self.ensure_vector_reg(a0)
+        loc1 = self.ensure_vector_reg(a1)
+        resloc = self.force_allocate_vector_reg(op)
+        return [resloc, loc0, loc1, imm(size)]
 
-    #    integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
-    #    aligned = False
-    #    self.perform_discard(op, [base_loc, ofs_loc, value_loc,
-    #                             imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
-
-    #prepare_vec_setarrayitem_raw = _prepare_vec_setarrayitem
-    #prepare_vec_setarrayitem_gc = _prepare_vec_setarrayitem
-    #prepare_vec_raw_store = _prepare_vec_setarrayitem
-
-    #def prepare_vec_arith(self, op):
-    #    lhs = op.getarg(0)
-    #    assert isinstance(op, VectorOp)
-    #    size = op.bytesize
-    #    args = op.getarglist()
-    #    loc1 = self.make_sure_var_in_reg(op.getarg(1), args)
-    #    loc0 = self.xrm.force_result_in_reg(op, op.getarg(0), args)
-    #    self.perform(op, [loc0, loc1, imm(size)], loc0)
-
-    #prepare_vec_int_add = prepare_vec_arith
+    prepare_vec_int_add = prepare_vec_arith
     #prepare_vec_int_sub = prepare_vec_arith
     #prepare_vec_int_mul = prepare_vec_arith
     #prepare_vec_float_add = prepare_vec_arith
     #prepare_vec_float_sub = prepare_vec_arith
     #prepare_vec_float_mul = prepare_vec_arith
     #prepare_vec_float_truediv = prepare_vec_arith
-    #del prepare_vec_arith
+    del prepare_vec_arith
+
+    def _prepare_vec_store(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        a2 = op.getarg(2)
+        baseloc = self.ensure_reg(a0)
+        ofsloc = self.ensure_reg(a1)
+        valueloc = self.ensure_vector_reg(a2)
+
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        aligned = False
+        return [baseloc, ofsloc, valueloc,
+                imm(itemsize), imm(ofs), imm(integer), imm(aligned)]
+
+    prepare_vec_setarrayitem_raw = _prepare_vec_store
+    prepare_vec_setarrayitem_gc = _prepare_vec_store
+    prepare_vec_raw_store = _prepare_vec_store
+    del _prepare_vec_store
+
 
     #def prepare_vec_arith_unary(self, op):
     #    lhs = op.getarg(0)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -1,5 +1,6 @@
 import py
 
+from hypothesis import given, note, strategies as st
 from rpython.jit.metainterp.warmspot import ll_meta_interp, get_stats
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.jit.codewriter.policy import StopAtXPolicy
@@ -23,6 +24,8 @@
 def free(mem):
     lltype.free(mem, flavor='raw')
 
+integers_64bit = st.integers(min_value=-2**63, max_value=2**63-1)
+
 class VectorizeTests:
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -37,21 +40,26 @@
                               type_system=self.type_system,
                               vec=vec, vec_all=vec_all)
 
-    @py.test.mark.parametrize('i',[3,4,5,6,7,8,9,50])
-    def test_vectorize_simple_load_arith_store_int_add_index(self,i):
+    @given(st.lists(integers_64bit, min_size=5, max_size=50),
+           st.lists(integers_64bit, min_size=5, max_size=50))
+    def test_vector_simple(self, la, lb):
         myjitdriver = JitDriver(greens = [],
                                 reds = 'auto',
                                 vectorize=True)
+        i = min(len(la), len(lb))
+        la = la[:i]
+        lb = lb[:i]
+        bc = i*rffi.sizeof(rffi.SIGNED)
+        vc = alloc_raw_storage(bc, zero=True)
+        size = rffi.sizeof(rffi.SIGNED)
         def f(d):
-            bc = d*rffi.sizeof(rffi.SIGNED)
             va = alloc_raw_storage(bc, zero=True)
             vb = alloc_raw_storage(bc, zero=True)
-            vc = alloc_raw_storage(bc, zero=True)
             x = 1
             for i in range(d):
-                j = i*rffi.sizeof(rffi.SIGNED)
-                raw_storage_setitem(va, j, rffi.cast(rffi.SIGNED,i))
-                raw_storage_setitem(vb, j, rffi.cast(rffi.SIGNED,i))
+                j = i*size
+                raw_storage_setitem(va, j, rffi.cast(rffi.SIGNED,la[i]))
+                raw_storage_setitem(vb, j, rffi.cast(rffi.SIGNED,lb[i]))
             i = 0
             while i < bc:
                 myjitdriver.jit_merge_point()
@@ -59,17 +67,15 @@
                 b = raw_storage_getitem(rffi.SIGNED,vb,i)
                 c = a+b
                 raw_storage_setitem(vc, i, rffi.cast(rffi.SIGNED,c))
-                i += 1*rffi.sizeof(rffi.SIGNED)
-            res = 0
-            for i in range(d):
-                res += raw_storage_getitem(rffi.SIGNED,vc,i*rffi.sizeof(rffi.SIGNED))
+                i += 1*size
 
             free_raw_storage(va)
             free_raw_storage(vb)
-            free_raw_storage(vc)
-            return res
-        res = self.meta_interp(f, [i])
-        assert res == f(i)
+        self.meta_interp(f, [i])
+        for p in range(i):
+            c = raw_storage_getitem(rffi.SIGNED,vc,p*size)
+            assert intmask(la[p] + lb[p]) == c
+        free_raw_storage(vc)
 
     @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,131,142,143])
     def test_vectorize_array_get_set(self,i):

From pypy.commits at gmail.com  Thu Jun 16 12:11:39 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 09:11:39 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: move pto.c_tp_itemsize
 assignment for str type to earlier point in cpyext initialization code
Message-ID: <5762cfbb.4dd11c0a.c5c55.ffff9354@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85192:d2e03a332a5b
Date: 2016-06-16 19:05 +0300
http://bitbucket.org/pypy/pypy/changeset/d2e03a332a5b/

Log:	move pto.c_tp_itemsize assignment for str type to earlier point in
	cpyext initialization code

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1202,8 +1202,6 @@
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
-            if space.is_w(w_type, space.w_str):
-                pto.c_tp_itemsize = 1
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -657,6 +657,8 @@
         pto.c_tp_dealloc = llhelper(
             subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
+    if space.is_w(w_type, space.w_str):
+        pto.c_tp_itemsize = 1
     # buffer protocol
     setup_buffer_procs(space, w_type, pto)
 

From pypy.commits at gmail.com  Thu Jun 16 12:11:41 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 09:11:41 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: translation fix
Message-ID: <5762cfbd.4f8d1c0a.a3c2.5343@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85193:c62ce292c86c
Date: 2016-06-16 19:06 +0300
http://bitbucket.org/pypy/pypy/changeset/c62ce292c86c/

Log:	translation fix

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -219,10 +219,10 @@
     set to NULL, a memory exception is set, and -1 is returned.
     """
     # XXX always create a new string so far
-    py_str = rffi.cast(PyStringObject, ref[0])
-    if pyobj_has_w_obj(py_str):
+    if pyobj_has_w_obj(ref[0]):
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
+    py_str = rffi.cast(PyStringObject, ref[0])
     try:
         py_newstr = new_empty_str(space, newsize)
     except MemoryError:

From pypy.commits at gmail.com  Thu Jun 16 14:51:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Jun 2016 11:51:00 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: pass arguments to a function
Message-ID: <5762f514.c5301c0a.d1f00.ffffa928@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85194:666baf5dc59b
Date: 2016-06-16 19:49 +0100
http://bitbucket.org/pypy/pypy/changeset/666baf5dc59b/

Log:	pass arguments to a function

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -20,8 +20,6 @@
 
 def c_compile(cfilenames, eci, outputfilename):
     self = rpy_platform
-    self.cfilenames = cfilenames
-    ext = so_ext
     self.libraries = list(eci.libraries)
     self.include_dirs = list(eci.include_dirs)
     self.library_dirs = list(eci.library_dirs)
@@ -47,16 +45,18 @@
         for framework in self.frameworks:
             self.link_extra += ['-framework', framework]
 
-    self.outputfilename = py.path.local(outputfilename).new(ext=ext)
-    self.eci = eci
+    outputfilename = py.path.local(outputfilename).new(ext=so_ext)
     import distutils.errors
-    basename = self.outputfilename.new(ext='')
+    basename = outputfilename.new(ext='')
     data = ''
     try:
         saved_environ = os.environ.copy()
         c = stdoutcapture.Capture(mixed_out_err=True)
         try:
-            _build(self)
+            _build(
+                cfilenames, outputfilename,
+                list(eci.compile_extra), self.link_extra,
+                self.include_dirs, self.libraries, self.library_dirs)
         finally:
             # workaround for a distutils bugs where some env vars can
             # become longer and longer every time it is used
@@ -75,23 +75,22 @@
     except:
         print >>sys.stderr, data
         raise
-    return self.outputfilename
+    return outputfilename
 
-def _build(self):
+def _build(cfilenames, outputfilename, compile_extra, link_extra,
+        include_dirs, libraries, library_dirs):
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler) # XXX
     compiler.spawn = log_spawned_cmd(compiler.spawn)
     objects = []
-    for cfile in self.cfilenames:
+    for cfile in cfilenames:
         cfile = py.path.local(cfile)
-        compile_extra = self.compile_extra[:]
-
         old = cfile.dirpath().chdir()
         try:
             res = compiler.compile([cfile.basename],
-                                    include_dirs=self.eci.include_dirs,
+                                    include_dirs=include_dirs,
                                     extra_preargs=compile_extra)
             assert len(res) == 1
             cobjfile = py.path.local(res[0])
@@ -100,7 +99,8 @@
         finally:
             old.chdir()
 
-    compiler.link_shared_object(objects, str(self.outputfilename),
-        libraries=self.eci.libraries,
-        extra_preargs=self.link_extra,
-        library_dirs=self.eci.library_dirs)
+    compiler.link_shared_object(
+        objects, str(outputfilename),
+        libraries=libraries,
+        extra_preargs=link_extra,
+        library_dirs=library_dirs)

From pypy.commits at gmail.com  Thu Jun 16 14:52:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 11:52:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.3.1 for
 changeset 7e8df3df9641
Message-ID: <5762f55c.441ac20a.67e79.6263@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85195:3feb181deba9
Date: 2016-06-16 21:49 +0300
http://bitbucket.org/pypy/pypy/changeset/3feb181deba9/

Log:	Added tag release-pypy2.7-v5.3.1 for changeset 7e8df3df9641

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1

From pypy.commits at gmail.com  Thu Jun 16 15:01:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 12:01:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: update repackage script for 5.3.1
Message-ID: <5762f772.949a1c0a.34c95.ffffd66e@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85196:79376977379b
Date: 2016-06-15 22:10 +0300
http://bitbucket.org/pypy/pypy/changeset/79376977379b/

Log:	update repackage script for 5.3.1

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,9 +1,9 @@
 # Edit these appropriately before running this script
 maj=5
 min=3
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1

From pypy.commits at gmail.com  Thu Jun 16 15:19:24 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 12:19:24 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update for bugfix release
 pypy2.7-v5.3.1
Message-ID: <5762fbbc.430ac20a.33013.ffffa51c@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r759:86fc69961a2b
Date: 2016-06-16 22:18 +0300
http://bitbucket.org/pypy/pypy.org/changeset/86fc69961a2b/

Log:	update for bugfix release pypy2.7-v5.3.1

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.
    
 
    We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
    
 
@@ -113,22 +113,22 @@
 degrees of being up-to-date.
    • 
 
 
    
-
    
-
    Python2.7 compatible PyPy 5.3
    
+
    
+
    Python2.7 compatible PyPy 5.3.1
    
 
@@ -138,7 +138,7 @@
 
    Warning: this is an alpha release supporting the Python 3.3 language.
 It's also known to be (sometimes much) slower than PyPy 2.
    
 
    
@@ -239,7 +239,7 @@
 
  • Get the source code.  The following packages contain the source at
 the same revision as the above binaries:
    
 
 
    Or you can checkout the current trunk using Mercurial (the trunk
 usually works and is of course more up-to-date):
    
@@ -392,6 +392,19 @@
 24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
 f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
    • +

    pypy2.7-v5.3.1 md5:

    +
    +0ff0e50e9595448d882fe94ab8667993  pypy2-v5.3.1-linux32.tar.bz2
+41979b51bd5d8f9d6475b6478cf38992  pypy2-v5.3.1-linux64.tar.bz2
+0f929b98566b154473a50820a3a6cbcf  pypy2-v5.3.1-linux-armel.tar.bz2
+06ff729d3e30a9787ede69f327534d13  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+be714716068e0e0c215e897f7c45ab34  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+ae7c15c8b831847f359f673e3461b4e6  pypy2-v5.3.1-osx64.tar.bz2
+a74f104c25aeb69d2eb2cdce01a2cb02  pypy2-v5.3.1-s390x.tar.bz2
+2ebc87d24018c60cbf339de039dfecb0  pypy2-v5.3.1-src.tar.bz2
+a1bbafb57c26de5aae8700cd7667a86b  pypy2-v5.3.1-src.zip
+38e7d4dd33ea636cc7faebdd94ef6cb4  pypy2-v5.3.1-win32.zip
+

    pypy3.3-v5.2-alpha md5:

     1176464541dff42e685bf8a9bb393796  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -422,6 +435,19 @@
 18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
 076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
    +

    pypy2.7-5.3.1 sha1:

    +
    +46dcd486ce2acbdf1815b29d70295ad305b667c5  pypy2-v5.3.1-linux32.tar.bz2
+a06eba349f6346dd6ab4c47f9dcccb0b42986478  pypy2-v5.3.1-linux64.tar.bz2
+60f33190bae1ac774f461b1da8f974ba1e8c0c24  pypy2-v5.3.1-linux-armel.tar.bz2
+3fd2fa66437ce72c179ea76522407cde74456e36  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+3f7f2aea02e90c6f4a5da00588cd06fdb74aa406  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+1d866608f21a465f58ec6f9003f837262f6f7b1a  pypy2-v5.3.1-osx64.tar.bz2
+08a31fe87ea99864780db86bdb7d7fb9029dc54c  pypy2-v5.3.1-s390x.tar.bz2
+3abd0c4d133fde7198bf81b15b7786e4e3de9f9f  pypy2-v5.3.1-src.tar.bz2
+fbfaba410f5031041907a28452cfd5e46b8a7336  pypy2-v5.3.1-src.zip
+2963d6b1a87f284dfc0336cbec4c37eeb1918f41  pypy2-v5.3.1-win32.zip
+

    pypy3.3-v5.2-alpha sha1:

     03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -447,6 +473,19 @@
 09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
 32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
    +

    pypy2.7-5.3.1 sha256:

    +
    +da69f4280b288e524387103eaa3eb4d036965724c3e546da27135c15a77bd2eb  pypy2-v5.3.1-linux32.tar.bz2
+6d0e8b14875b76b1e77f06a2ee3f1fb5015a645a951ba7a7586289344d4d9c22  pypy2-v5.3.1-linux64.tar.bz2
+0425f2022c35ef7f0bb3d2b854c5bcbe500b1aba511a0d83581ba6c784913961  pypy2-v5.3.1-linux-armel.tar.bz2
+b4859496099bde4b17c1e56cc5749dcdcd25b4c68fde1d2ea426de84130e84cc  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+5c93eb3c54fbb2c7d7332f775a096671512e590565e6051196bbc5039c5033b5  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+7a242d7373b4f18c7f5fe6c2fe6f15e2a405d9adf1f4f934c89b875e60ac5def  pypy2-v5.3.1-osx64.tar.bz2
+61262f0727ee04b225761b59ce270a64fae9b986d22405a93340f05d0d5c0e0e  pypy2-v5.3.1-s390x.tar.bz2
+31a52bab584abf3a0f0defd1bf9a29131dab08df43885e7eeddfc7dc9b71836e  pypy2-v5.3.1-src.tar.bz2
+7eab4a8435583750088001d88371cd0314999b67f26f32ab9d006802f0beec2e  pypy2-v5.3.1-src.zip
+d83477e2c5f032ebd8c7f47afce03dc8adbeb41a3c74f7db50d9de317dcf3a4a  pypy2-v5.3.1-win32.zip
+

    pypy3.3-v5.2-alpha sha256:

     351aec101bdedddae7ea1b63845a5654b1a95fc9393894ef84a66749f6945f17  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,12 +14,13 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release — **PyPy2.7 v5.3.0** — (`what's new in PyPy2.7?`_ )
+* the Python2.7 compatible release — **PyPy2.7 v5.3.0** — (`what's new in PyPy2.7?`_  and release note for `PyPy2.7-v5.3.1`_)
 * the Python3.3 compatible release — **PyPy3.3 v5.2-alpha** — (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)
 
 .. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.0.html
+.. _PyPy2.7-v5.3.1: http://doc.pypy.org/en/latest/release-pypy2.7-v5.3.1.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html
 
 
@@ -73,7 +74,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 5.3
+Python2.7 compatible PyPy 5.3.1
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -92,18 +93,18 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-win32.zip
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64.tar.bz2
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0+-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
@@ -203,7 +204,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.3.0/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.3.1/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -259,9 +260,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.3.0-src.tar.bz2`__ (sources)
+   * `pypy2-v5.3.1-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.0-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::
@@ -428,6 +429,19 @@
     24add66f18ab2213c9e44af0ada61085  pypy2-v5.3.0-src.zip
     f6197adf58bfa32bcb18451289da1c7c  pypy2-v5.3.0-win32.zip
 
+pypy2.7-v5.3.1 md5::
+
+    0ff0e50e9595448d882fe94ab8667993  pypy2-v5.3.1-linux32.tar.bz2
+    41979b51bd5d8f9d6475b6478cf38992  pypy2-v5.3.1-linux64.tar.bz2
+    0f929b98566b154473a50820a3a6cbcf  pypy2-v5.3.1-linux-armel.tar.bz2
+    06ff729d3e30a9787ede69f327534d13  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+    be714716068e0e0c215e897f7c45ab34  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+    ae7c15c8b831847f359f673e3461b4e6  pypy2-v5.3.1-osx64.tar.bz2
+    a74f104c25aeb69d2eb2cdce01a2cb02  pypy2-v5.3.1-s390x.tar.bz2
+    2ebc87d24018c60cbf339de039dfecb0  pypy2-v5.3.1-src.tar.bz2
+    a1bbafb57c26de5aae8700cd7667a86b  pypy2-v5.3.1-src.zip
+    38e7d4dd33ea636cc7faebdd94ef6cb4  pypy2-v5.3.1-win32.zip
+
 pypy3.3-v5.2-alpha md5::
 
     1176464541dff42e685bf8a9bb393796  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -459,6 +473,19 @@
     18a81c46b3d0ecf7e186c19a7d302986a5b15a83  pypy2-v5.3.0-src.zip
     076251ba3b44435dc11867dab00f392b058bdc7c  pypy2-v5.3.0-win32.zip
 
+pypy2.7-5.3.1 sha1::
+
+    46dcd486ce2acbdf1815b29d70295ad305b667c5  pypy2-v5.3.1-linux32.tar.bz2
+    a06eba349f6346dd6ab4c47f9dcccb0b42986478  pypy2-v5.3.1-linux64.tar.bz2
+    60f33190bae1ac774f461b1da8f974ba1e8c0c24  pypy2-v5.3.1-linux-armel.tar.bz2
+    3fd2fa66437ce72c179ea76522407cde74456e36  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+    3f7f2aea02e90c6f4a5da00588cd06fdb74aa406  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+    1d866608f21a465f58ec6f9003f837262f6f7b1a  pypy2-v5.3.1-osx64.tar.bz2
+    08a31fe87ea99864780db86bdb7d7fb9029dc54c  pypy2-v5.3.1-s390x.tar.bz2
+    3abd0c4d133fde7198bf81b15b7786e4e3de9f9f  pypy2-v5.3.1-src.tar.bz2
+    fbfaba410f5031041907a28452cfd5e46b8a7336  pypy2-v5.3.1-src.zip
+    2963d6b1a87f284dfc0336cbec4c37eeb1918f41  pypy2-v5.3.1-win32.zip
+
 pypy3.3-v5.2-alpha sha1::
 
     03c1181f3866b977598e56b4263c8373d3f3a712  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2
@@ -484,6 +511,19 @@
     09914006c2c5c394bb6f847f6eb9c9322737c7238e7ca482c5a12c9e2ef76a58  pypy2-v5.3.0-src.zip
     32a9e5286fc344165f63b529a9f84e521e9368e717c583488115654676428a20  pypy2-v5.3.0-win32.zip
 
+pypy2.7-5.3.1 sha256::
+
+    da69f4280b288e524387103eaa3eb4d036965724c3e546da27135c15a77bd2eb  pypy2-v5.3.1-linux32.tar.bz2
+    6d0e8b14875b76b1e77f06a2ee3f1fb5015a645a951ba7a7586289344d4d9c22  pypy2-v5.3.1-linux64.tar.bz2
+    0425f2022c35ef7f0bb3d2b854c5bcbe500b1aba511a0d83581ba6c784913961  pypy2-v5.3.1-linux-armel.tar.bz2
+    b4859496099bde4b17c1e56cc5749dcdcd25b4c68fde1d2ea426de84130e84cc  pypy2-v5.3.1-linux-armhf-raring.tar.bz2
+    5c93eb3c54fbb2c7d7332f775a096671512e590565e6051196bbc5039c5033b5  pypy2-v5.3.1-linux-armhf-raspbian.tar.bz2
+    7a242d7373b4f18c7f5fe6c2fe6f15e2a405d9adf1f4f934c89b875e60ac5def  pypy2-v5.3.1-osx64.tar.bz2
+    61262f0727ee04b225761b59ce270a64fae9b986d22405a93340f05d0d5c0e0e  pypy2-v5.3.1-s390x.tar.bz2
+    31a52bab584abf3a0f0defd1bf9a29131dab08df43885e7eeddfc7dc9b71836e  pypy2-v5.3.1-src.tar.bz2
+    7eab4a8435583750088001d88371cd0314999b67f26f32ab9d006802f0beec2e  pypy2-v5.3.1-src.zip
+    d83477e2c5f032ebd8c7f47afce03dc8adbeb41a3c74f7db50d9de317dcf3a4a  pypy2-v5.3.1-win32.zip
+
 pypy3.3-v5.2-alpha sha256::
 
     351aec101bdedddae7ea1b63845a5654b1a95fc9393894ef84a66749f6945f17  pypy3.3-v5.2.0-alpha1-linux32.tar.bz2

From pypy.commits at gmail.com  Thu Jun 16 16:28:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 13:28:29 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: fix one more place
Message-ID: <57630bed.49c51c0a.2c64e.0837@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r760:d884fbe4ae57
Date: 2016-06-16 23:28 +0300
http://bitbucket.org/pypy/pypy.org/changeset/d884fbe4ae57/

Log:	fix one more place

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.
    
 
    We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
    
 
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,7 +14,7 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release — **PyPy2.7 v5.3.0** — (`what's new in PyPy2.7?`_  and release note for `PyPy2.7-v5.3.1`_)
+* the Python2.7 compatible release — **PyPy2.7 v5.3.1** — (`what's new in PyPy2.7?`_  and release note for `PyPy2.7-v5.3.1`_)
 * the Python3.3 compatible release — **PyPy3.3 v5.2-alpha** — (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)

From pypy.commits at gmail.com  Thu Jun 16 16:37:05 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 13:37:05 -0700 (PDT)
Subject: [pypy-commit] pypy pyfile-tell: add failing test for syncing
 ftell(fp) and fid.tell()
Message-ID: <57630df1.6372c20a.3456f.491c@mx.google.com>

Author: Matti Picus 
Branch: pyfile-tell
Changeset: r85197:4941e3ed2258
Date: 2016-06-16 22:51 +0300
http://bitbucket.org/pypy/pypy/changeset/4941e3ed2258/

Log:	add failing test for syncing ftell(fp) and fid.tell()

diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+

From pypy.commits at gmail.com  Thu Jun 16 16:37:07 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 13:37:07 -0700 (PDT)
Subject: [pypy-commit] pypy pyfile-tell: document why the test fails,
 need to call flush_buffers in PyFile_AsFile
Message-ID: <57630df3.692dc20a.489bc.ffffb79f@mx.google.com>

Author: Matti Picus 
Branch: pyfile-tell
Changeset: r85198:b23ad2aee931
Date: 2016-06-16 23:33 +0300
http://bitbucket.org/pypy/pypy/changeset/b23ad2aee931/

Log:	document why the test fails, need to call flush_buffers in
	PyFile_AsFile

diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -142,5 +142,9 @@
             t_py = fid.tell()
             assert t_py == 80
             t_c = module.get_c_tell(fid)
+        # XXX currently fails since rlib/streamio BufferingInputStream has 
+        #     an internal readahead buffer self.buf and its self.pos can be 
+        #     out of sync with the underlying c-level FILE * (it's not called
+        #     a BufferingInputStream for nothing)
         assert t_c == t_py
 

From pypy.commits at gmail.com  Thu Jun 16 16:37:09 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 13:37:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix link
Message-ID: <57630df5.4e4ec20a.a91f1.2e2d@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85199:a756c02c565a
Date: 2016-06-16 23:33 +0300
http://bitbucket.org/pypy/pypy/changeset/a756c02c565a/

Log:	fix link

diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
--- a/pypy/doc/release-pypy2.7-v5.3.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -7,7 +7,7 @@
 
 Thanks to those who reported the issues.
 
-.. _issues http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
 
 What is PyPy?
 =============

From pypy.commits at gmail.com  Thu Jun 16 16:39:30 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 16 Jun 2016 13:39:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix visit_list
Message-ID: <57630e82.a91cc20a.d7d7b.ffffcca1@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85200:8d7c7f8d2ee3
Date: 2016-06-16 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/8d7c7f8d2ee3/

Log:	Fix visit_list

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1018,6 +1018,7 @@
         self.use_next_block(end)
 
     def _visit_list_or_tuple(self, node, elts, ctx, op):
+        #TODO
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
             seen_star = False
@@ -1040,7 +1041,6 @@
         if ctx == ast.Load:
             self.emit_op_arg(op, elt_count)
     
-    #TODO
     def _visit_starunpack(self, node, elts, ctx, single_op, innter_op, outer_op):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
@@ -1076,7 +1076,12 @@
 
     def visit_List(self, l):
         self.update_position(l.lineno)
-        self._visit_list_or_tuple(l, l.elts, l.ctx, ops.BUILD_LIST)
+        if l.ctx == ast.Store:
+            self._visit_list_or_tuple_assignment(l, l.elts)
+        elif l.ctx == ast.Load:
+            self._visit_list_or_tuple_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST)
+        else
+            self.visit_sequence(l.elts)
 
     def visit_Dict(self, d):
         self.update_position(d.lineno)

From pypy.commits at gmail.com  Thu Jun 16 16:43:50 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 16 Jun 2016 13:43:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix visit_tuple
Message-ID: <57630f86.4f941c0a.efeff.ffffeb4e@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85201:d3a8be3eee83
Date: 2016-06-16 22:42 +0200
http://bitbucket.org/pypy/pypy/changeset/d3a8be3eee83/

Log:	Fix visit_tuple

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1072,7 +1072,12 @@
 
     def visit_Tuple(self, tup):
         self.update_position(tup.lineno)
-        self._visit_list_or_tuple(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE)
+        if l.ctx == ast.Store:
+            self._visit_list_or_tuple_assignment(l, l.elts)
+        elif l.ctx == ast.Load:
+            self._visit_list_or_tuple_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE)
+        else
+            self.visit_sequence(l.elts)
 
     def visit_List(self, l):
         self.update_position(l.lineno)

From pypy.commits at gmail.com  Thu Jun 16 17:39:35 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Thu, 16 Jun 2016 14:39:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Adapt _visit_list_or_tuple_assignment
Message-ID: <57631c97.cf981c0a.7bc2c.ffffccfb@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85202:430c5404f5b9
Date: 2016-06-16 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/430c5404f5b9/

Log:	Adapt _visit_list_or_tuple_assignment

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1017,7 +1017,7 @@
         ifexp.orelse.walkabout(self)
         self.use_next_block(end)
 
-    def _visit_list_or_tuple(self, node, elts, ctx, op):
+    def _visit_list_or_tuple_starunpack(self, node, elts, ctx, op):
         #TODO
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
@@ -1041,7 +1041,8 @@
         if ctx == ast.Load:
             self.emit_op_arg(op, elt_count)
     
-    def _visit_starunpack(self, node, elts, ctx, single_op, innter_op, outer_op):
+    #_visit_starunpack
+    def _visit_list_or_tuple_assignment(self, node, elts, ctx, single_op, innter_op, outer_op):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
             seen_star = False
@@ -1061,8 +1062,8 @@
             if not seen_star:
                 self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
         self.visit_sequence(elts)
-        if ctx == ast.Load:
-            self.emit_op_arg(op, elt_count)
+        #if ctx == ast.Load:
+        #    self.emit_op_arg(op, elt_count)
 
     def visit_Starred(self, star):
         if star.ctx != ast.Store:
@@ -1076,7 +1077,7 @@
             self._visit_list_or_tuple_assignment(l, l.elts)
         elif l.ctx == ast.Load:
             self._visit_list_or_tuple_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE)
-        else
+        else:
             self.visit_sequence(l.elts)
 
     def visit_List(self, l):
@@ -1085,7 +1086,7 @@
             self._visit_list_or_tuple_assignment(l, l.elts)
         elif l.ctx == ast.Load:
             self._visit_list_or_tuple_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST)
-        else
+        else:
             self.visit_sequence(l.elts)
 
     def visit_Dict(self, d):

From pypy.commits at gmail.com  Thu Jun 16 20:27:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Jun 2016 17:27:50 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Don't use RPython logging
Message-ID: <57634406.a758c20a.2867f.1999@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85203:73843504475e
Date: 2016-06-17 01:26 +0100
http://bitbucket.org/pypy/pypy/changeset/73843504475e/

Log:	Don't use RPython logging

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -1,18 +1,10 @@
 import os
-import sys
 import py
-from rpython.translator.platform import log, CompilationError
-from rpython.translator.tool import stdoutcapture
+from rpython.translator.platform import log
 from rpython.translator.platform.distutils_platform import DistutilsPlatform
 
 rpy_platform = DistutilsPlatform()
 
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        log.execute(' '.join(cmd))
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
 if os.name != 'nt':
     so_ext = 'so'
 else:
@@ -46,35 +38,18 @@
             self.link_extra += ['-framework', framework]
 
     outputfilename = py.path.local(outputfilename).new(ext=so_ext)
-    import distutils.errors
-    basename = outputfilename.new(ext='')
-    data = ''
+    saved_environ = os.environ.copy()
     try:
-        saved_environ = os.environ.copy()
-        c = stdoutcapture.Capture(mixed_out_err=True)
-        try:
-            _build(
-                cfilenames, outputfilename,
-                list(eci.compile_extra), self.link_extra,
-                self.include_dirs, self.libraries, self.library_dirs)
-        finally:
-            # workaround for a distutils bugs where some env vars can
-            # become longer and longer every time it is used
-            for key, value in saved_environ.items():
-                if os.environ.get(key) != value:
-                    os.environ[key] = value
-            foutput, foutput = c.done()
-            data = foutput.read()
-            if data:
-                fdump = basename.new(ext='errors').open("wb")
-                fdump.write(data)
-                fdump.close()
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        raise CompilationError('', data)
-    except:
-        print >>sys.stderr, data
-        raise
+        _build(
+            cfilenames, outputfilename,
+            list(eci.compile_extra), self.link_extra,
+            self.include_dirs, self.libraries, self.library_dirs)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
     return outputfilename
 
 def _build(cfilenames, outputfilename, compile_extra, link_extra,
@@ -83,7 +58,6 @@
     from distutils import sysconfig
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler) # XXX
-    compiler.spawn = log_spawned_cmd(compiler.spawn)
     objects = []
     for cfile in cfilenames:
         cfile = py.path.local(cfile)

From pypy.commits at gmail.com  Thu Jun 16 21:58:07 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Jun 2016 18:58:07 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: stop using rpy_platform
 attributes as local variables
Message-ID: <5763592f.d11b1c0a.39976.40ef@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85204:43826d85eaaa
Date: 2016-06-17 02:57 +0100
http://bitbucket.org/pypy/pypy/changeset/43826d85eaaa/

Log:	stop using rpy_platform attributes as local variables

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -12,38 +12,37 @@
 
 def c_compile(cfilenames, eci, outputfilename):
     self = rpy_platform
-    self.libraries = list(eci.libraries)
-    self.include_dirs = list(eci.include_dirs)
-    self.library_dirs = list(eci.library_dirs)
-    self.compile_extra = list(eci.compile_extra)
-    self.link_extra = list(eci.link_extra)
-    self.frameworks = list(eci.frameworks)
+    libraries = list(eci.libraries)
+    include_dirs = list(eci.include_dirs)
+    library_dirs = list(eci.library_dirs)
+    compile_extra = list(eci.compile_extra)
+    link_extra = list(eci.link_extra)
+    frameworks = list(eci.frameworks)
     if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
-        if 'm' not in self.libraries:
-            self.libraries.append('m')
-        if 'pthread' not in self.libraries:
-            self.libraries.append('pthread')
+        if 'm' not in libraries:
+            libraries.append('m')
+        if 'pthread' not in libraries:
+            libraries.append('pthread')
     if self.name == 'win32':
-        self.link_extra += ['/DEBUG'] # generate .pdb file
+        link_extra += ['/DEBUG'] # generate .pdb file
     if self.name == 'darwin':
         # support Fink & Darwinports
         for s in ('/sw/', '/opt/local/'):
-            if s + 'include' not in self.include_dirs and \
+            if s + 'include' not in include_dirs and \
                 os.path.exists(s + 'include'):
-                self.include_dirs.append(s + 'include')
-            if s + 'lib' not in self.library_dirs and \
-                os.path.exists(s + 'lib'):
-                self.library_dirs.append(s + 'lib')
-        for framework in self.frameworks:
-            self.link_extra += ['-framework', framework]
+                include_dirs.append(s + 'include')
+            if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
+                library_dirs.append(s + 'lib')
+        for framework in frameworks:
+            link_extra += ['-framework', framework]
 
     outputfilename = py.path.local(outputfilename).new(ext=so_ext)
     saved_environ = os.environ.copy()
     try:
         _build(
             cfilenames, outputfilename,
-            list(eci.compile_extra), self.link_extra,
-            self.include_dirs, self.libraries, self.library_dirs)
+            compile_extra, link_extra,
+            include_dirs, libraries, library_dirs)
     finally:
         # workaround for a distutils bugs where some env vars can
         # become longer and longer every time it is used
@@ -64,8 +63,7 @@
         old = cfile.dirpath().chdir()
         try:
             res = compiler.compile([cfile.basename],
-                                    include_dirs=include_dirs,
-                                    extra_preargs=compile_extra)
+                include_dirs=include_dirs, extra_preargs=compile_extra)
             assert len(res) == 1
             cobjfile = py.path.local(res[0])
             assert cobjfile.check()

From pypy.commits at gmail.com  Thu Jun 16 23:06:16 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Jun 2016 20:06:16 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Don't use DistutilsPlatform
Message-ID: <57636928.0654c20a.1573.1464@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85206:2cd0dfca3da2
Date: 2016-06-17 04:05 +0100
http://bitbucket.org/pypy/pypy/changeset/2cd0dfca3da2/

Log:	Don't use DistutilsPlatform

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -1,9 +1,6 @@
 import os
 import py
-from rpython.translator.platform import log
-from rpython.translator.platform.distutils_platform import DistutilsPlatform
-
-rpy_platform = DistutilsPlatform()
+from sys import platform
 
 if os.name != 'nt':
     so_ext = 'so'
@@ -18,15 +15,14 @@
     include_dirs = include_dirs or []
     libraries = libraries or []
     library_dirs = library_dirs or []
-    self = rpy_platform
-    if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
+    if not platform in ('win32', 'darwin', 'cygwin'): # xxx
         if 'm' not in libraries:
             libraries.append('m')
         if 'pthread' not in libraries:
             libraries.append('pthread')
-    if self.name == 'win32':
+    if platform == 'win32':
         link_extra = link_extra + ['/DEBUG'] # generate .pdb file
-    if self.name == 'darwin':
+    if platform == 'darwin':
         # support Fink & Darwinports
         for s in ('/sw/', '/opt/local/'):
             if (s + 'include' not in include_dirs

From pypy.commits at gmail.com  Thu Jun 16 23:06:14 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Jun 2016 20:06:14 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Don't use ECI
Message-ID: <57636926.e40bc30a.b819.fffff724@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85205:cc4219028500
Date: 2016-06-17 03:46 +0100
http://bitbucket.org/pypy/pypy/changeset/cc4219028500/

Log:	Don't use ECI

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -10,31 +10,30 @@
 else:
     so_ext = 'dll'
 
-def c_compile(cfilenames, eci, outputfilename):
+def c_compile(cfilenames, outputfilename,
+        compile_extra=None, link_extra=None,
+        include_dirs=None, libraries=None, library_dirs=None):
+    compile_extra = compile_extra or []
+    link_extra = link_extra or []
+    include_dirs = include_dirs or []
+    libraries = libraries or []
+    library_dirs = library_dirs or []
     self = rpy_platform
-    libraries = list(eci.libraries)
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    compile_extra = list(eci.compile_extra)
-    link_extra = list(eci.link_extra)
-    frameworks = list(eci.frameworks)
     if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
         if 'm' not in libraries:
             libraries.append('m')
         if 'pthread' not in libraries:
             libraries.append('pthread')
     if self.name == 'win32':
-        link_extra += ['/DEBUG'] # generate .pdb file
+        link_extra = link_extra + ['/DEBUG'] # generate .pdb file
     if self.name == 'darwin':
         # support Fink & Darwinports
         for s in ('/sw/', '/opt/local/'):
-            if s + 'include' not in include_dirs and \
-                os.path.exists(s + 'include'):
+            if (s + 'include' not in include_dirs
+                    and os.path.exists(s + 'include')):
                 include_dirs.append(s + 'include')
             if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
                 library_dirs.append(s + 'lib')
-        for framework in frameworks:
-            link_extra += ['-framework', framework]
 
     outputfilename = py.path.local(outputfilename).new(ext=so_ext)
     saved_environ = os.environ.copy()
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,7 +7,6 @@
 from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -41,17 +40,17 @@
         files.append(filename)
     return files
 
-def create_so(modname, include_dirs,
-        source_strings=None,
-        source_files=None,
-        **kwds):
+def create_so(modname, include_dirs, source_strings=None, source_files=None,
+        compile_extra=None, link_extra=None, libraries=None):
     dirname = (udir/uniquemodulename('module')).ensure(dir=1)
     if source_strings:
         assert not source_files
         files = convert_sources_to_files(source_strings, dirname)
         source_files = files
-    eci = ExternalCompilationInfo(include_dirs=include_dirs, **kwds)
-    soname = c_compile(source_files, eci, outputfilename=str(dirname/modname))
+    soname = c_compile(source_files, outputfilename=str(dirname/modname),
+        compile_extra=compile_extra, link_extra=link_extra,
+        include_dirs=include_dirs,
+        libraries=libraries)
     return soname
 
 def compile_extension_module(space, modname, include_dirs=[],
@@ -66,31 +65,32 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    kwds = {}
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        libraries = [api_library]
         # '%s' undefined; assuming extern returning int
-        kwds["compile_extra"] = ["/we4013"]
+        compile_extra = ["/we4013"]
         # prevent linking with PythonXX.lib
         w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
                               (space.int_w(w_maj), space.int_w(w_min))]
-    elif sys.platform == 'darwin':
-        kwds["link_files"] = [str(api_library + '.dylib')]
     else:
-        kwds["link_files"] = [str(api_library + '.so')]
+        libraries = []
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"] = ["-Werror", "-g", "-O0", "-fPIC"]
-            kwds["link_extra"] = ["-g"]
+            compile_extra = ["-Werror", "-g", "-O0", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
 
     modname = modname.split('.')[-1]
     soname = create_so(modname,
             include_dirs=api.include_dirs + include_dirs,
             source_files=source_files,
             source_strings=source_strings,
-            **kwds)
+            compile_extra=compile_extra,
+            link_extra=link_extra,
+            libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
@@ -108,22 +108,24 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    kwds = {}
     if sys.platform == 'win32':
-        kwds["compile_extra"] = ["/we4013"]
-        kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
+        compile_extra = ["/we4013"]
+        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
     elif sys.platform == 'darwin':
+        compile_extra = link_extra = None
         pass
     elif sys.platform.startswith('linux'):
-        kwds["compile_extra"] = [
+        compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
+        link_extra = None
 
     modname = modname.split('.')[-1]
     soname = create_so(modname,
             include_dirs=[space.include_dir] + include_dirs,
             source_files=source_files,
             source_strings=source_strings,
-            **kwds)
+            compile_extra=compile_extra,
+            link_extra=link_extra)
     return str(soname)
 
 def freeze_refcnts(self):

From pypy.commits at gmail.com  Fri Jun 17 02:16:20 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 23:16:20 -0700 (PDT)
Subject: [pypy-commit] pypy pyfile-tell: add flush_buffers to PyFile_AsFile
Message-ID: <576395b4.41561c0a.b85dc.6e23@mx.google.com>

Author: Matti Picus 
Branch: pyfile-tell
Changeset: r85207:48f59a7416c9
Date: 2016-06-17 09:10 +0300
http://bitbucket.org/pypy/pypy/changeset/48f59a7416c9/

Log:	add flush_buffers to PyFile_AsFile

diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -142,9 +142,5 @@
             t_py = fid.tell()
             assert t_py == 80
             t_c = module.get_c_tell(fid)
-        # XXX currently fails since rlib/streamio BufferingInputStream has 
-        #     an internal readahead buffer self.buf and its self.pos can be 
-        #     out of sync with the underlying c-level FILE * (it's not called
-        #     a BufferingInputStream for nothing)
         assert t_c == t_py
 

From pypy.commits at gmail.com  Fri Jun 17 02:16:24 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 23:16:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge pyfile-tell which syncs w_file
 with the c-level FILE* before returning FILE*
Message-ID: <576395b8.cc9d1c0a.fd717.7fa6@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85209:2ccda01df68b
Date: 2016-06-17 09:12 +0300
http://bitbucket.org/pypy/pypy/changeset/2ccda01df68b/

Log:	merge pyfile-tell which syncs w_file with the c-level FILE* before
	returning FILE*

diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+

From pypy.commits at gmail.com  Fri Jun 17 02:16:26 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 23:16:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <576395ba.c91d1c0a.b30a9.ffffa10d@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85210:d8cafae12463
Date: 2016-06-17 09:15 +0300
http://bitbucket.org/pypy/pypy/changeset/d8cafae12463/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -35,3 +35,5 @@
 Simplify handling of interp-level tests and make it more forward-
 compatible.
 
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile

From pypy.commits at gmail.com  Fri Jun 17 02:16:22 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Jun 2016 23:16:22 -0700 (PDT)
Subject: [pypy-commit] pypy pyfile-tell: close branch to be merged
Message-ID: <576395b6.cc9d1c0a.fd717.7fa4@mx.google.com>

Author: Matti Picus 
Branch: pyfile-tell
Changeset: r85208:b1432381e6fd
Date: 2016-06-17 09:11 +0300
http://bitbucket.org/pypy/pypy/changeset/b1432381e6fd/

Log:	close branch to be merged


From pypy.commits at gmail.com  Fri Jun 17 08:02:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 05:02:38 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add revdb.breakpoint()
Message-ID: <5763e6de.c72d1c0a.eed07.3ea2@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85211:30bf0d9e0ee7
Date: 2016-06-17 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/30bf0d9e0ee7/

Log:	Add revdb.breakpoint()

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -52,6 +52,10 @@
     """
     _change_time('f', time_delta, callback, arg_string)
 
+ at specialize.arg(0)
+def breakpoint(callback, arg_string):
+    _change_time('k', 1, callback, arg_string)
+
 @specialize.arg(1)
 def jump_in_time(target_time, callback, arg_string, exact=True):
     """For RPython debug commands: the debugger should run the
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -861,6 +861,19 @@
     case 'b':       /* go non exact */
         cmd_go(time >= 1 ? time : 1, callback, arg, mode);
         abort();    /* unreachable */
+
+    case 'k':       /* breakpoint */
+        assert(time > 0);
+        if (stopped_time != 0) {
+            fprintf(stderr, "revdb.breakpoint(): cannot be called from a "
+                            "debug command\n");
+            exit(1);
+        }
+        rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + time;
+        invoke_after_forward = callback;
+        invoke_argument = arg;
+        break;
+
     default:
         abort();    /* unreachable */
     }
@@ -892,6 +905,7 @@
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
+    rpy_revdb.unique_id_break = 0;
     tracked_object = new_object;
     rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
     return rpy_revdb.unique_id_seen;

From pypy.commits at gmail.com  Fri Jun 17 11:26:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 08:26:05 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Update track_object() to allow
 tracking any number of objects
Message-ID: <5764168d.c5301c0a.4d59b.6179@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85212:4ab10ccbb305
Date: 2016-06-17 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/4ab10ccbb305/

Log:	Update track_object() to allow tracking any number of objects

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -91,29 +91,20 @@
     """
     return llop.revdb_get_unique_id(lltype.SignedLongLong, x)
 
-def track_object(unique_id):
+def track_object(unique_id, callback):
     """Track the creation of the object given by its unique_id, which must
     be in the future (i.e. >= currently_created_objects()).  Call this
     before go_forward().  If go_forward() goes over the creation of this
-    object, then afterwards, get_tracked_object() returns the object.
-    Going forward is also interrupted at the following stop point.
-    Object tracking is lost by jump_in_time(), like everything else.
+    object, then 'callback(gcref)' is called.  Careful in callback(),
+    gcref is not fully initialized and should not be immediately read from,
+    only stored for later.  The purpose of callback() is to possibly
+    call track_object() again to track the next object, and/or to call
+    breakpoint().  Note: object tracking remains activated until one of:
+    (1) we reach the creation time in go_forward(); (2) we call
+    track_object() to track a different object; (3) we call jump_in_time().
     """
-    llop.revdb_track_object(lltype.Void, unique_id)
-
- at specialize.arg(0)
-def get_tracked_object(Class=llmemory.GCREF):   # or an RPython class
-    """Get the tracked object if it was created during the last go_forward().
-    Otherwise, returns None.  (Note: this API is minimal: to get an
-    object from its unique id, you need first to search backward for a
-    time where currently_created_objects() is lower than the unique_id,
-    then use track_object() and go_forward() to come back.  You can't
-    really track several objects, only one.)
-    """
-    x = llop.revdb_get_tracked_object(llmemory.GCREF)
-    if Class is llmemory.GCREF:
-        return x
-    return cast_gcref_to_instance(Class, x)
+    ll_callback = llhelper(_CALLBACK_GCREF_FNPTR, callback)
+    llop.revdb_track_object(lltype.Void, unique_id, ll_callback)
 
 
 # ____________________________________________________________
@@ -132,6 +123,8 @@
     return callback_wrapper
 _CALLBACK_ARG_FNPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)],
                                                  lltype.Void))
+_CALLBACK_GCREF_FNPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
+                                                   lltype.Void))
 
 
 class RegisterDebugCommand(ExtRegistryEntry):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -573,7 +573,6 @@
     'revdb_identityhash':   LLOp(),
     'revdb_get_unique_id':  LLOp(sideeffects=False),
     'revdb_track_object':   LLOp(),
-    'revdb_get_tracked_object': LLOp(sideeffects=False),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -900,23 +900,22 @@
     }
 }
 
-static void *tracked_object;
+static void (*unique_id_callback)(void *);
 
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
     rpy_revdb.unique_id_break = 0;
-    tracked_object = new_object;
-    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
+    unique_id_callback(new_object);
     return rpy_revdb.unique_id_seen;
 }
 
 RPY_EXTERN
-void rpy_reverse_db_track_object(long long unique_id)
+void rpy_reverse_db_track_object(long long unique_id, void callback(void *))
 {
     if (stopped_uid <= 0) {
         fprintf(stderr, "stopped_uid should not be <= 0\n");
-        exit(1);
+        return;
     }
     if (unique_id <= 0) {
         printf("cannot track a prebuilt or debugger-created object\n");
@@ -926,15 +925,10 @@
         printf("cannot track the creation of an object already created\n");
         return;
     }
+    assert(callback != NULL);
+    unique_id_callback = callback;
     rpy_revdb.unique_id_break = unique_id;
-    tracked_object = NULL;
 }
 
-RPY_EXTERN
-void *rpy_reverse_db_get_tracked_object(void)
-{
-    return tracked_object;
-}
-
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -96,11 +96,8 @@
 #define OP_REVDB_GET_UNIQUE_ID(x, r)                                    \
     r = ((struct pypy_header0 *)x)->h_uid
 
-#define OP_REVDB_TRACK_OBJECT(uid, r)                                   \
-    rpy_reverse_db_track_object(uid)
-
-#define OP_REVDB_GET_TRACKED_OBJECT(r)                                  \
-    r = rpy_reverse_db_get_tracked_object()
+#define OP_REVDB_TRACK_OBJECT(uid, callback, r)                         \
+    rpy_reverse_db_track_object(uid, callback)
 
 RPY_EXTERN void rpy_reverse_db_flush(void);
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
@@ -113,8 +110,8 @@
                                            RPyString *arg);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
-RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id);
-RPY_EXTERN void *rpy_reverse_db_get_tracked_object(void);
+RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id,
+                                            void callback(void *));
 
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -6,6 +6,8 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rlib import objectmodel, revdb
 from rpython.rlib.rarithmetic import intmask
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.lltypesystem import lltype, llmemory
 """
 These tests require pexpect (UNIX-only).
 http://pexpect.sourceforge.net/
@@ -314,6 +316,10 @@
         def _nothing(arg):
             pass
         #
+        def callback_track_obj(gcref):
+            revdb.send_output("callback_track_obj\n")
+            dbstate.gcref = gcref
+        #
         def blip(cmdline):
             revdb.send_output('<<<' + cmdline + '>>>\n')
             if cmdline == 'oops':
@@ -345,11 +351,14 @@
                     revdb.currently_created_objects()))
             if cmdline.startswith('track-object '):
                 uid = int(cmdline[len('track-object '):])
-                revdb.track_object(uid)
+                dbstate.gcref = lltype.nullptr(llmemory.GCREF.TO)
+                revdb.track_object(uid, callback_track_obj)
             if cmdline == 'get-tracked-object':
-                obj = revdb.get_tracked_object(Stuff)
-                revdb.send_output('None\n' if obj is None else
-                                  ('obj.x=%d\n' % obj.x))
+                if dbstate.gcref:
+                    revdb.send_output('got obj.x=%d\n' % (
+                        cast_gcref_to_instance(Stuff, dbstate.gcref).x,))
+                else:
+                    revdb.send_output('none\n')
             if cmdline == 'first-created-uid':
                 revdb.send_output('first-created-uid=%d\n' % (
                     revdb.first_created_object_uid(),))
@@ -503,32 +512,24 @@
         child.expectx('<<>>\r\n' % object_id_3rd +
                       'blipped\r\n'
                       '(1)$ ')
-        prev_time = 1
-        for i in [1, 2, 3]:
+        for i in [1, 2]:
             child.sendline('r get-tracked-object')
             child.expectx('<<>>\r\n'
-                          'None\r\n'
+                          'none\r\n'
                           'blipped\r\n'
-                          '(%d)$ ' % prev_time)
-            child.sendline('__forward %d' % (i - prev_time))
-            child.expectx('(%d)$ ' % i)
-            prev_time = i
-            child.sendline('r print-id')
-            child.expect(re.escape('<<>>\r\n')
-                         + r'obj.x=%d (\d+) (\d+)' % (1000 + prev_time - 1)
-                         + re.escape('\r\n'
-                                     'blipped\r\n'
-                                     '(%d)$ ' % prev_time))
+                          '(%d)$ ' % i)
+            child.sendline('__forward 1')
+            child.expectx('(%d)$ ' % (i + 1))
         child.sendline('r get-tracked-object')
         child.expectx('<<>>\r\n'
-                      'obj.x=1002\r\n'
+                      'got obj.x=1002\r\n'
                       'blipped\r\n'
                       '(3)$ ')
         child.sendline('__go 3')
         child.expectx('(3)$ ')
         child.sendline('r get-tracked-object')
         child.expectx('<<>>\r\n'
-                      'None\r\n'
+                      'none\r\n'
                       'blipped\r\n'
                       '(3)$ ')
         #
@@ -559,13 +560,21 @@
         child.expectx('<<>>\r\n' % object_id_2nd +
                       'blipped\r\n'
                       '(1)$ ')
-        child.sendline('__forward 5')
-        child.expectx('(2)$ ')
+        child.sendline('__forward 2')
+        child.expectx('(3)$ ')
         child.sendline('r get-tracked-object')
         child.expectx('<<>>\r\n'
-                      'obj.x=1001\r\n'
+                      'got obj.x=1001\r\n'
                       'blipped\r\n'
-                      '(2)$ ')
+                      '(3)$ ')
+        child.sendline('__forward 1')
+        child.expectx('At end.\r\n'
+                      '(3)$ ')
+        child.sendline('r get-tracked-object')
+        child.expectx('<<>>\r\n'
+                      'none\r\n'
+                      'blipped\r\n'
+                      '(3)$ ')
 
     def test_first_created_uid(self):
         child = self.replay()

From pypy.commits at gmail.com  Fri Jun 17 11:26:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 08:26:07 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix tests.  Improve safety
Message-ID: <5764168f.89dec20a.a1a5d.4e45@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85213:10e58efcb71e
Date: 2016-06-17 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/10e58efcb71e/

Log:	Fix tests. Improve safety

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -364,6 +364,10 @@
 static void disable_io(rpy_revdb_t *dinfo)
 {
     *dinfo = rpy_revdb;   /* save the complete struct */
+    dinfo->saved_exc[0] = pypy_g_ExcData.ed_exc_type;
+    dinfo->saved_exc[1] = pypy_g_ExcData.ed_exc_value;
+    pypy_g_ExcData.ed_exc_type = NULL;
+    pypy_g_ExcData.ed_exc_value = NULL;
     rpy_revdb.buf_p = NULL;
     rpy_revdb.buf_limit = NULL;
     flag_io_disabled = 1;
@@ -374,12 +378,19 @@
     uint64_t v1, v2;
     flag_io_disabled = 0;
 
+    if (pypy_g_ExcData.ed_exc_type != NULL) {
+        printf("Command crashed with %.*s\n",
+               (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
+               pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
+    }
     /* restore the complete struct, with the exception of '*_break' */
     v1 = rpy_revdb.stop_point_break;
     v2 = rpy_revdb.unique_id_break;
     rpy_revdb = *dinfo;
     rpy_revdb.stop_point_break = v1;
     rpy_revdb.unique_id_break = v2;
+    pypy_g_ExcData.ed_exc_type = dinfo->saved_exc[0];
+    pypy_g_ExcData.ed_exc_value = dinfo->saved_exc[1];
 }
 
 /* generated by RPython */
@@ -418,27 +429,12 @@
 static void execute_rpy_function(void func(RPyString *), RPyString *arg)
 {
     rpy_revdb_t dinfo;
-    void *saved_t = pypy_g_ExcData.ed_exc_type;
-    void *saved_v = pypy_g_ExcData.ed_exc_value;
-    pypy_g_ExcData.ed_exc_type = NULL;
-    pypy_g_ExcData.ed_exc_value = NULL;
     disable_io(&dinfo);
     invoke_after_forward = NULL;
     invoke_argument = NULL;
-
-    if (setjmp(jmp_buf_cancel_execution) == 0) {
-
+    if (setjmp(jmp_buf_cancel_execution) == 0)
         func(arg);
-
-        if (pypy_g_ExcData.ed_exc_type != NULL) {
-            printf("Command crashed with %.*s\n",
-                   (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
-                   pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
-        }
-    }
     enable_io(&dinfo);
-    pypy_g_ExcData.ed_exc_type = saved_t;
-    pypy_g_ExcData.ed_exc_value = saved_v;
 }
 
 struct action_s {
@@ -905,8 +901,12 @@
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
+    rpy_revdb_t dinfo;
     rpy_revdb.unique_id_break = 0;
-    unique_id_callback(new_object);
+    disable_io(&dinfo);
+    if (setjmp(jmp_buf_cancel_execution) == 0)
+        unique_id_callback(new_object);
+    enable_io(&dinfo);
     return rpy_revdb.unique_id_seen;
 }
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -19,6 +19,7 @@
     char *buf_p, *buf_limit;
     uint64_t stop_point_seen, stop_point_break;
     uint64_t unique_id_seen, unique_id_break;
+    void *saved_exc[2];
 } rpy_revdb_t;
 
 RPY_EXTERN rpy_revdb_t rpy_revdb;
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -236,10 +236,11 @@
                 print op
                 lst.append(op + '??')   # create a new string here
             for x in lst:
-                print revdb.creation_time_of(x)
+                print revdb.get_unique_id(x)
             return 9
         compile(cls, main, [], backendopt=False)
-        assert run(cls, 'abc d ef') == 'abc\nd\nef\n0\n0\n1\n2\n3\n'
+        assert run(cls, 'abc d ef') == ('abc\nd\nef\n'
+                                        '3\n0\n12\n15\n17\n')
         rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef'])
         assert rdb.number_of_stop_points() == 3
 

From pypy.commits at gmail.com  Fri Jun 17 12:24:03 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 17 Jun 2016 09:24:03 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Don't add random libraries
Message-ID: <57642423.56311c0a.59323.0f26@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85214:2b1f8306e59a
Date: 2016-06-17 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/2b1f8306e59a/

Log:	Don't add random libraries

diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
--- a/pypy/module/cpyext/test/support.py
+++ b/pypy/module/cpyext/test/support.py
@@ -15,11 +15,6 @@
     include_dirs = include_dirs or []
     libraries = libraries or []
     library_dirs = library_dirs or []
-    if not platform in ('win32', 'darwin', 'cygwin'): # xxx
-        if 'm' not in libraries:
-            libraries.append('m')
-        if 'pthread' not in libraries:
-            libraries.append('pthread')
     if platform == 'win32':
         link_extra = link_extra + ['/DEBUG'] # generate .pdb file
     if platform == 'darwin':

From pypy.commits at gmail.com  Fri Jun 17 12:48:55 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 17 Jun 2016 09:48:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix wrong names,
 change parameters in assisting methods in visit_List and visit_Tuple
Message-ID: <576429f7.665ec20a.bcd5e.54ba@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85215:d54ef7518fde
Date: 2016-06-17 18:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d54ef7518fde/

Log:	Fix wrong names, change parameters in assisting methods in
	visit_List and visit_Tuple

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1073,19 +1073,19 @@
 
     def visit_Tuple(self, tup):
         self.update_position(tup.lineno)
-        if l.ctx == ast.Store:
-            self._visit_list_or_tuple_assignment(l, l.elts)
-        elif l.ctx == ast.Load:
-            self._visit_list_or_tuple_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE)
+        if tup.ctx == ast.Store:
+            self._visit_list_or_tuple_assignment(tup, tup.elts, tup.ctx)
+        elif tup.ctx == ast.Load:
+            self._visit_list_or_tuple_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
         else:
-            self.visit_sequence(l.elts)
+            self.visit_sequence(tup.elts)
 
     def visit_List(self, l):
         self.update_position(l.lineno)
         if l.ctx == ast.Store:
-            self._visit_list_or_tuple_assignment(l, l.elts)
+            self._visit_list_or_tuple_assignment(l, l.elts, l.ctx)
         elif l.ctx == ast.Load:
-            self._visit_list_or_tuple_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST)
+            self._visit_list_or_tuple_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
         else:
             self.visit_sequence(l.elts)
 

From pypy.commits at gmail.com  Fri Jun 17 14:06:36 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Fri, 17 Jun 2016 11:06:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Write starunpack and assignment assist
 methods for tuple and list
Message-ID: <57643c2c.094ac20a.6bfe0.7250@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85216:bad2cce14a59
Date: 2016-06-17 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/bad2cce14a59/

Log:	Write starunpack and assignment assist methods for tuple and list

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1017,8 +1017,33 @@
         ifexp.orelse.walkabout(self)
         self.use_next_block(end)
 
-    def _visit_list_or_tuple_starunpack(self, node, elts, ctx, op):
-        #TODO
+    def _visit_list_or_tuple_starunpack(self, node, elts, ctx, single_op, inner_op, outer_op):
+        elt_count = len(elts) if elts else 0
+        seen_star = 0
+        elt_subitems = 0
+        for i in range(elt_count):
+            elt = elts[i]
+            is_starred = isinstance(elt, ast.Starred)
+            if is_starred:
+                if seen_star:
+                    self.emit_op_arg(inner_op, seen_star)
+                    seen_star = 0
+                    elt_subitems += 1
+                elt.value.walkabout(self)
+                elt_subitems += 1
+            else:
+                elt.walkabout(self)
+                seen_star += 1
+        if elt_subitems:
+            if seen_star:
+                self.emit_op_arg(inner_op, seen_star)
+                elt_subitems += 1
+            self.emit_op_arg(outer_op, elt_subitems)
+        else:
+            self.emit_op_arg(single_op, seen_star)
+    
+    #_visit_starunpack
+    def _visit_list_or_tuple_assignment(self, node, elts, ctx):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
             seen_star = False
@@ -1038,32 +1063,6 @@
             if not seen_star:
                 self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
         self.visit_sequence(elts)
-        if ctx == ast.Load:
-            self.emit_op_arg(op, elt_count)
-    
-    #_visit_starunpack
-    def _visit_list_or_tuple_assignment(self, node, elts, ctx, single_op, innter_op, outer_op):
-        elt_count = len(elts) if elts else 0
-        if ctx == ast.Store:
-            seen_star = False
-            for i in range(elt_count):
-                elt = elts[i]
-                is_starred = isinstance(elt, ast.Starred)
-                if is_starred and not seen_star:
-                    if i >= 1 << 8 or elt_count - i - 1 >= (C_INT_MAX >> 8):
-                        self.error("too many expressions in star-unpacking "
-                                   "assignment", node)
-                    self.emit_op_arg(ops.UNPACK_EX,
-                                     i + ((elt_count - i - 1) << 8))
-                    seen_star = True
-                    elts[i] = elt.value
-                elif is_starred:
-                    self.error("two starred expressions in assignment", node)
-            if not seen_star:
-                self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
-        self.visit_sequence(elts)
-        #if ctx == ast.Load:
-        #    self.emit_op_arg(op, elt_count)
 
     def visit_Starred(self, star):
         if star.ctx != ast.Store:

From pypy.commits at gmail.com  Fri Jun 17 15:16:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 12:16:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Change the model: now that the
 basics work, a more flexible model involving
Message-ID: <57644c99.82e01c0a.f2b9e.0759@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85217:829c0b70159d
Date: 2016-06-17 21:17 +0200
http://bitbucket.org/pypy/pypy/changeset/829c0b70159d/

Log:	Change the model: now that the basics work, a more flexible model
	involving a regular Python controller program makes more sense

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -1,11 +1,13 @@
 import sys
 from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_longlong
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.annlowlevel import llhelper, hlstr
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.lltypesystem import rffi
 
 
 def stop_point():
@@ -20,9 +22,9 @@
 def register_debug_command(command, lambda_func):
     """Register the extra RPython-implemented debug command."""
 
-def send_output(string):
-    """For RPython debug commands: writes the string to stdout."""
-    llop.revdb_send_output(lltype.Void, string)
+def send_answer(cmd, arg1=0, arg2=0, arg3=0, extra=""):
+    """For RPython debug commands: writes an answer block to stdout"""
+    llop.revdb_send_answer(lltype.Void, cmd, arg1, arg2, arg3, extra)
 
 def current_time():
     """For RPython debug commands: returns the current time."""
@@ -33,41 +35,11 @@
     this is the target time at which we'll stop going forward."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'b')
 
-def most_recent_fork():
-    """For RPython debug commands: returns the time of the most
-    recent fork.  Going back to that time is fast; going back to a time
-    just before is slow."""
-    return llop.revdb_get_value(lltype.SignedLongLong, 'f')
-
 def total_time():
     """For RPython debug commands: returns the total time (measured
     as the total number of stop-points)."""
     return llop.revdb_get_value(lltype.SignedLongLong, 't')
 
- at specialize.arg(1)
-def go_forward(time_delta, callback, arg_string):
-    """For RPython debug commands: tells that after this function finishes,
-    the debugger should run the 'forward ' command and then
-    invoke the 'callback' with no argument.
-    """
-    _change_time('f', time_delta, callback, arg_string)
-
- at specialize.arg(0)
-def breakpoint(callback, arg_string):
-    _change_time('k', 1, callback, arg_string)
-
- at specialize.arg(1)
-def jump_in_time(target_time, callback, arg_string, exact=True):
-    """For RPython debug commands: the debugger should run the
-    'go ' command.  This will reset the memory and fork again,
-    so you can't save any RPython state and read it back.  You can only
-    encode the state you want to save into a string.  In the reloaded
-    process, 'callback(arg_string)' is called.  If 'exact' is False, go to
-    the fork point before target_time but don't go_forward to exactly
-    target_time afterwards.
-    """
-    _change_time('g' if exact else 'b', target_time, callback, arg_string)
-
 def currently_created_objects():
     """For RPython debug commands: returns the current value of
     the object creation counter.  All objects created so far have
@@ -75,11 +47,13 @@
     unique id greater or equal."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'u')
 
-def first_created_object_uid():
-    """Returns the creation number of the first object dynamically created
-    by the program.  Older objects are either prebuilt or created before
-    the first stop point."""
-    return llop.revdb_get_value(lltype.SignedLongLong, '1')
+ at specialize.arg(1)
+def go_forward(time_delta, callback):
+    """For RPython debug commands: tells that after this function finishes,
+    the debugger should run the 'forward ' command and then
+    invoke the 'callback' with no argument.
+    """
+    _change_time('f', time_delta, callback)
 
 @specialize.argtype(0)
 def get_unique_id(x):
@@ -111,10 +85,10 @@
 
 
 @specialize.arg(2)
-def _change_time(mode, time, callback, arg_string):
+def _change_time(mode, time, callback):
     callback_wrapper = _make_callback(callback)
     ll_callback = llhelper(_CALLBACK_ARG_FNPTR, callback_wrapper)
-    llop.revdb_change_time(lltype.Void, mode, time, ll_callback, arg_string)
+    llop.revdb_change_time(lltype.Void, mode, time, ll_callback)
 
 @specialize.memo()
 def _make_callback(callback):
@@ -125,16 +99,23 @@
                                                  lltype.Void))
 _CALLBACK_GCREF_FNPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
                                                    lltype.Void))
+_CMDPTR = rffi.CStructPtr('rpy_revdb_command_s',
+                          ('cmd', rffi.INT),
+                          ('arg1', lltype.SignedLongLong),
+                          ('arg2', lltype.SignedLongLong),
+                          ('arg3', lltype.SignedLongLong))
 
 
 class RegisterDebugCommand(ExtRegistryEntry):
     _about_ = register_debug_command
 
-    def compute_result_annotation(self, s_command, s_lambda_func):
+    def compute_result_annotation(self, s_command_num, s_lambda_func):
         from rpython.annotator import model as annmodel
-        command = s_command.const
+        from rpython.rtyper import llannotation
+
+        command_num = s_command_num.const
         lambda_func = s_lambda_func.const
-        assert isinstance(command, str)
+        assert isinstance(command_num, int)
         t = self.bookkeeper.annotator.translator
         if t.config.translation.reverse_debugger:
             func = lambda_func()
@@ -142,10 +123,12 @@
                 cmds = t.revdb_commands
             except AttributeError:
                 cmds = t.revdb_commands = []
-            cmds.append((command, func))
+            cmds.append((command_num, func))
             s_func = self.bookkeeper.immutablevalue(func)
+            s_ptr1 = llannotation.SomePtr(ll_ptrtype=_CMDPTR)
+            s_ptr2 = llannotation.SomePtr(ll_ptrtype=rffi.CCHARP)
             self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                             s_func, [annmodel.s_Str0])
+                                             s_func, [s_ptr1, s_ptr2])
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -566,13 +566,13 @@
     'instrument_count':     LLOp(),
 
     'revdb_stop_point':     LLOp(),
-    'revdb_send_output':    LLOp(),
+    'revdb_send_answer':    LLOp(),
     'revdb_change_time':    LLOp(),
     'revdb_get_value':      LLOp(sideeffects=False),
-    'revdb_set_value':      LLOp(),
-    'revdb_identityhash':   LLOp(),
-    'revdb_get_unique_id':  LLOp(sideeffects=False),
-    'revdb_track_object':   LLOp(),
+    ## 'revdb_set_value':      LLOp(),
+    ## 'revdb_identityhash':   LLOp(),
+    ## 'revdb_get_unique_id':  LLOp(sideeffects=False),
+    ## 'revdb_track_object':   LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/translator/c/src/g_prerequisite.h b/rpython/translator/c/src/g_prerequisite.h
--- a/rpython/translator/c/src/g_prerequisite.h
+++ b/rpython/translator/c/src/g_prerequisite.h
@@ -23,3 +23,7 @@
 # define RPY_LENGTH0     1       /* array decl [0] are bad */
 # define RPY_DUMMY_VARLENGTH     /* nothing */
 #endif
+
+#ifdef RPY_REVERSE_DEBUGGER
+#include "src-revdb/revdb_preinclude.h"
+#endif
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -1,7 +1,7 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.translator.c.support import cdecl
-from rpython.rlib import exports
+from rpython.rlib import exports, revdb
 
 
 def extra_files():
@@ -23,12 +23,13 @@
 
 
 def prepare_database(db):
-    FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void))
+    FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR,
+                                          rffi.CCHARP], lltype.Void))
 
     bk = db.translator.annotator.bookkeeper
     cmds = getattr(db.translator, 'revdb_commands', [])
 
-    array_names = lltype.malloc(rffi.CArray(rffi.CCHARP), len(cmds) + 1,
+    array_names = lltype.malloc(rffi.CArray(rffi.INT), len(cmds) + 1,
                                 flavor='raw', immortal=True, zero=True)
     array_funcs = lltype.malloc(rffi.CArray(FUNCPTR), len(cmds),
                                 flavor='raw', immortal=True, zero=True)
@@ -36,7 +37,8 @@
     for i, (name, func) in enumerate(cmds):
         fnptr = lltype.getfunctionptr(bk.getdesc(func).getuniquegraph())
         assert lltype.typeOf(fnptr) == FUNCPTR
-        array_names[i] = rffi.str2charp(name)
+        assert isinstance(name, int) and name != 0
+        array_names[i] = rffi.cast(rffi.INT, name)
         array_funcs[i] = fnptr
 
     exports.EXPORTS_obj2name[array_names._as_obj()] = 'rpy_revdb_command_names'
diff --git a/rpython/translator/revdb/src-revdb/ancillary.h b/rpython/translator/revdb/src-revdb/ancillary.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/src-revdb/ancillary.h
@@ -0,0 +1,123 @@
+/***************************************************************************
+ * libancillary - black magic on Unix domain sockets
+ * (C) Nicolas George
+ * ancillary.h - public header
+ ***************************************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef ANCILLARY_H__
+#define ANCILLARY_H__
+
+/***************************************************************************
+ * Start of the readable part.
+ ***************************************************************************/
+
+#define ANCIL_MAX_N_FDS 960
+/*
+ * Maximum number of fds that can be sent or received using the "esay"
+ * functions; this is so that all can fit in one page.
+ */
+
+extern int
+ancil_send_fds_with_buffer(int, const int *, unsigned, void *);
+/*
+ * ancil_send_fds_with_buffer(sock, n_fds, fds, buffer)
+ *
+ * Sends the file descriptors in the array pointed by fds, of length n_fds
+ * on the socket sock.
+ * buffer is a writeable memory area large enough to hold the required data
+ * structures.
+ * Returns: -1 and errno in case of error, 0 in case of success.
+ */
+
+extern int
+ancil_recv_fds_with_buffer(int, int *, unsigned, void *);
+/*
+ * ancil_recv_fds_with_buffer(sock, n_fds, fds, buffer)
+ *
+ * Receives *n_fds file descriptors into the array pointed by fds
+ * from the socket sock.
+ * buffer is a writeable memory area large enough to hold the required data
+ * structures.
+ * Returns: -1 and errno in case of error, the actual number of received fd
+ * in case of success
+ */
+
+#define ANCIL_FD_BUFFER(n) \
+    struct { \
+	struct cmsghdr h; \
+	int fd[n]; \
+    }
+/* ANCIL_FD_BUFFER(n)
+ *
+ * A structure type suitable to be used as buffer for n file descriptors.
+ * Requires .
+ * Example:
+ * ANCIL_FD_BUFFER(42) buffer;
+ * ancil_recv_fds_with_buffer(sock, 42, my_fds, &buffer);
+ */
+
+extern int
+ancil_send_fds(int, const int *, unsigned);
+/*
+ * ancil_send_fds(sock, n_fds, fds)
+ *
+ * Sends the file descriptors in the array pointed by fds, of length n_fds
+ * on the socket sock.
+ * n_fds must not be greater than ANCIL_MAX_N_FDS.
+ * Returns: -1 and errno in case of error, 0 in case of success.
+ */
+
+extern int
+ancil_recv_fds(int, int *, unsigned);
+/*
+ * ancil_recv_fds(sock, n_fds, fds)
+ *
+ * Receives *n_fds file descriptors into the array pointed by fds
+ * from the socket sock.
+ * *n_fds must not be greater than ANCIL_MAX_N_FDS.
+ * Returns: -1 and errno in case of error, the actual number of received fd
+ * in case of success.
+ */
+
+
+extern int
+ancil_send_fd(int, int);
+/* ancil_recv_fd(sock, fd);
+ *
+ * Sends the file descriptor fd on the socket sock.
+ * Returns : -1 and errno in case of error, 0 in case of success.
+ */
+
+extern int
+ancil_recv_fd(int, int *);
+/* ancil_send_fd(sock, &fd);
+ *
+ * Receives the file descriptor fd from the socket sock.
+ * Returns : -1 and errno in case of error, 0 in case of success.
+ */
+
+#endif /* ANCILLARY_H__ */
diff --git a/rpython/translator/revdb/src-revdb/fd_recv.c b/rpython/translator/revdb/src-revdb/fd_recv.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/src-revdb/fd_recv.c
@@ -0,0 +1,98 @@
+/***************************************************************************
+ * libancillary - black magic on Unix domain sockets
+ * (C) Nicolas George
+ * fd_send.c - receiving file descriptors
+ ***************************************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _XPG4_2 /* Solaris sucks */
+# define _XPG4_2
+#endif
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#if defined(__FreeBSD__)
+# include  /* FreeBSD sucks */
+#endif
+
+#include "ancillary.h"
+
+int
+ancil_recv_fds_with_buffer(int sock, int *fds, unsigned n_fds, void *buffer)
+{
+    struct msghdr msghdr;
+    char nothing;
+    struct iovec nothing_ptr;
+    struct cmsghdr *cmsg;
+    int i;
+
+    nothing_ptr.iov_base = ¬hing;
+    nothing_ptr.iov_len = 1;
+    msghdr.msg_name = NULL;
+    msghdr.msg_namelen = 0;
+    msghdr.msg_iov = ¬hing_ptr;
+    msghdr.msg_iovlen = 1;
+    msghdr.msg_flags = 0;
+    msghdr.msg_control = buffer;
+    msghdr.msg_controllen = sizeof(struct cmsghdr) + sizeof(int) * n_fds;
+    cmsg = CMSG_FIRSTHDR(&msghdr);
+    cmsg->cmsg_len = msghdr.msg_controllen;
+    cmsg->cmsg_level = SOL_SOCKET;
+    cmsg->cmsg_type = SCM_RIGHTS;
+    for(i = 0; i < n_fds; i++)
+	((int *)CMSG_DATA(cmsg))[i] = -1;
+    
+    if(recvmsg(sock, &msghdr, 0) < 0)
+	return(-1);
+    for(i = 0; i < n_fds; i++)
+	fds[i] = ((int *)CMSG_DATA(cmsg))[i];
+    n_fds = (cmsg->cmsg_len - sizeof(struct cmsghdr)) / sizeof(int);
+    return(n_fds);
+}
+
+#ifndef SPARE_RECV_FDS
+int
+ancil_recv_fds(int sock, int *fd, unsigned n_fds)
+{
+    ANCIL_FD_BUFFER(ANCIL_MAX_N_FDS) buffer;
+
+    assert(n_fds <= ANCIL_MAX_N_FDS);
+    return(ancil_recv_fds_with_buffer(sock, fd, n_fds, &buffer));
+}
+#endif /* SPARE_RECV_FDS */
+
+#ifndef SPARE_RECV_FD
+int
+ancil_recv_fd(int sock, int *fd)
+{
+    ANCIL_FD_BUFFER(1) buffer;
+
+    return(ancil_recv_fds_with_buffer(sock, fd, 1, &buffer) == 1 ? 0 : -1);
+}
+#endif /* SPARE_RECV_FD */
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "structdef.h"
 #include "forwarddecl.h"
@@ -175,70 +176,42 @@
 /* ------------------------------------------------------------ */
 
 
-/* How it works: the main process reads the RevDB file and
-   reconstructs the GC objects, in effect replaying their content, for
-   the complete duration of the original program.  During this
-   replaying, it forks a fixed number of frozen processes which sit
-   around, each keeping the version of the GC objects contents at some
-   known version.  We have one pipe for every frozen process, which
-   the frozen process is blocked reading.
-
-    [main process]
-        [frozen process 1]
-        [frozen process 2]
-            [debugging process]
-        [frozen process 3]
-        [frozen process 4]
-        ...
-        [frozen process n]
-
-   The main process's job, once reloading is finished, is only to
-   active debugging processes, one at a time.  To go to a specific
-   target time, it activates the right frozen process by sending
-   'target_time' over the corresponding pipe.  The frozen process
-   forks the debugging process, and the debugging process goes forward
-   until it reaches 'target_time'.
-
-   The debugging process is then interacting with the user on
-   stdin/stdout.
-
-   A few commands like 'go ' will cause the debugging
-   process to send the 'target_time' back over a signalling pipe to
-   the main process, and then finish.  The main process receives that
-   'target_time', and the loop closes: it activates the right frozen
-   process, which will go forward and re-enter interactive mode.
+/* How it works: we run the same executable with different flags to
+   run it in "replay" mode.  In this mode, it reads commands from
+   stdin (in binary format) and writes the results to stdout.
+   Notably, there is a command to ask it to fork, passing a new pair
+   of pipes to the forked copy as its new stdin/stdout.  This is how
+   we implement the illusion of going backward: we throw away the
+   current fork, start from an earlier fork, make a new fork again,
+   and go forward by the correct number of steps.  This is all
+   controlled by a pure Python wrapper that is roughly generic
+   (i.e. able to act as a debugger for any language).
 */
 
-#define NUM_FROZEN_PROCESSES   30
-#define STEP_RATIO             0.25
+#include "src-revdb/fd_recv.c"
 
-#define RD_SIDE   0
-#define WR_SIDE   1
+#define INIT_VERSION_NUMBER   0xd80100
 
-static int frozen_num_pipes = 0;
-static int frozen_pipes[NUM_FROZEN_PROCESSES][2];
-static uint64_t frozen_time[NUM_FROZEN_PROCESSES];
-static int frozen_pipe_signal[2];
+#define CMD_FORK     (-1)
+#define CMD_QUIT     (-2)
+#define CMD_FORWARD  (-3)
 
-enum { PK_MAIN_PROCESS, PK_FROZEN_PROCESS, PK_DEBUG_PROCESS };
-static unsigned char process_kind = PK_MAIN_PROCESS;
-static jmp_buf jmp_buf_cancel_execution;
-static uint64_t most_recent_fork;
-static uint64_t total_stop_points;
+#define ANSWER_INIT    (-20)
+#define ANSWER_STD     (-21)
+#define ANSWER_FORKED  (-22)
+#define ANSWER_AT_END  (-23)
+
+#define RDB_STDIN   0
+#define RDB_STDOUT  1
+
+typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, char *);
+
+static const char *rpy_rev_filename;
 static uint64_t stopped_time;
 static uint64_t stopped_uid;
-static uint64_t first_created_uid;
-
-static void (*invoke_after_forward)(RPyString *);
-static RPyString *invoke_argument;
-
-struct jump_in_time_s {
-    uint64_t target_time;
-    char mode;
-    void *callback;
-    size_t arg_length;
-};
-
+static uint64_t total_stop_points;
+static jmp_buf jmp_buf_cancel_execution;
+static void (*pending_after_forward)(void);
 
 static void attach_gdb(void)
 {
@@ -256,11 +229,9 @@
         ssize_t rsize = read(rpy_rev_fileno, buf + result, count_max - result);
         if (rsize <= 0) {
             if (rsize == 0)
-                fprintf(stderr, "[%d] RevDB file appears truncated\n",
-                        process_kind);
+                fprintf(stderr, "RevDB file appears truncated\n");
             else
-                fprintf(stderr, "[%d] RevDB file read error: %m\n",
-                        process_kind);
+                fprintf(stderr, "RevDB file read error: %m\n");
             exit(1);
         }
         result += rsize;
@@ -273,11 +244,63 @@
     (void)read_at_least(buf, count, count);
 }
 
+
+static void read_pipe(int fd, void *buf, ssize_t count)
+{
+    while (count > 0) {
+        ssize_t got = read(fd, buf, count);
+        if (got <= 0) {
+            fprintf(stderr, "subprocess: cannot read pipe %d\n", fd);
+            exit(1);
+        }
+        count -= got;
+        buf += got;
+    }
+}
+
+static void write_pipe(int fd, const void *buf, ssize_t count)
+{
+    while (count > 0) {
+        ssize_t wrote = write(fd, buf, count);
+        if (wrote <= 0) {
+            fprintf(stderr, "subprocess: cannot write to pipe %d\n", fd);
+            exit(1);
+        }
+        count -= wrote;
+        buf += wrote;
+    }
+}
+
+static void write_answer(int cmd, int64_t arg1, int64_t arg2, int64_t arg3)
+{
+    rpy_revdb_command_t c;
+    memset(&c, 0, sizeof(c));
+    c.cmd = cmd;
+    c.arg1 = arg1;
+    c.arg2 = arg2;
+    c.arg3 = arg3;
+    write_pipe(RDB_STDOUT, &c, sizeof(c));
+}
+
+static void answer_std(void)
+{
+    write_answer(ANSWER_STD, rpy_revdb.stop_point_seen,
+                 rpy_revdb.unique_id_seen, 0);
+}
+
+static void reopen_revdb_file(const char *filename)
+{
+    rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY);
+    if (rpy_rev_fileno < 0) {
+        fprintf(stderr, "Can't open file '%s': %m\n", filename);
+        exit(1);
+    }
+}
+
 static void setup_replay_mode(int *argc_p, char **argv_p[])
 {
     int argc = *argc_p;
     char **argv = *argv_p;
-    char *filename;
     rdb_header_t h;
     char input[16];
     ssize_t count;
@@ -286,20 +309,15 @@
         fprintf(stderr, "syntax: %s --revdb-replay \n", argv[0]);
         exit(2);
     }
-    filename = argv[2];
-
-    rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY);
-    if (rpy_rev_fileno < 0) {
-        fprintf(stderr, "Can't open file '%s': %m\n", filename);
-        exit(1);
-    }
+    rpy_rev_filename = argv[2];
+    reopen_revdb_file(rpy_rev_filename);
 
     assert(RPY_RDB_REPLAY == 1);
 
     read_all(input, strlen(RDB_SIGNATURE));
     if (strncmp(input, RDB_SIGNATURE, strlen(RDB_SIGNATURE)) != 0) {
         fprintf(stderr, "'%s' is not a RevDB file (or wrong platform)\n",
-                filename);
+                rpy_rev_filename);
         exit(1);
     }
     fprintf(stderr, "%s", RDB_SIGNATURE);
@@ -321,7 +339,7 @@
             read(rpy_rev_fileno, &total_stop_points,
                  sizeof(uint64_t)) != sizeof(uint64_t) ||
             lseek(rpy_rev_fileno, count, SEEK_SET) != count) {
-        fprintf(stderr, "%s: %m\n", filename);
+        fprintf(stderr, "%s: %m\n", rpy_rev_filename);
         exit(1);
     }
 
@@ -330,10 +348,12 @@
     rpy_revdb.stop_point_break = 1;
     rpy_revdb.unique_id_seen = 1;
 
-    if (pipe(frozen_pipe_signal) < 0) {
-        perror("pipe");
-        exit(1);
-    }
+    write_answer(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points, 0);
+    pending_after_forward = &answer_std;
+
+    /* ignore the SIGCHLD signals so that child processes don't become
+       zombies */
+    signal(SIGCHLD, SIG_IGN);
 }
 
 RPY_EXTERN
@@ -379,9 +399,10 @@
     flag_io_disabled = 0;
 
     if (pypy_g_ExcData.ed_exc_type != NULL) {
-        printf("Command crashed with %.*s\n",
-               (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
-               pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
+        fprintf(stderr, "Command crashed with %.*s\n",
+                (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length),
+                pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items);
+        exit(1);
     }
     /* restore the complete struct, with the exception of '*_break' */
     v1 = rpy_revdb.stop_point_break;
@@ -393,24 +414,7 @@
     pypy_g_ExcData.ed_exc_value = dinfo->saved_exc[1];
 }
 
-/* generated by RPython */
-extern char *rpy_revdb_command_names[];
-extern void (*rpy_revdb_command_funcs[])(RPyString *);
-
-static RPyString *make_rpy_string(size_t length)
-{
-    RPyString *s = malloc(sizeof(RPyString) + length);
-    if (s == NULL) {
-        fprintf(stderr, "out of memory for a string of %llu chars\n",
-                (unsigned long long)length);
-        exit(1);
-    }
-    /* xxx assumes Boehm here for now */
-    memset(s, 0, sizeof(RPyString));
-    RPyString_Size(s) = length;
-    return s;
-}
-
+/*
 static void execute_rpy_function(void func(RPyString *), RPyString *arg);
 
 static void execute_rpy_command(long index, char *arguments)
@@ -425,152 +429,27 @@
 
     execute_rpy_function(rpy_revdb_command_funcs[index], s);
 }
+*/
 
-static void execute_rpy_function(void func(RPyString *), RPyString *arg)
+static void execute_rpy_function(rpy_revdb_command_fn func,
+                                 rpy_revdb_command_t *cmd,
+                                 char *extra)
 {
     rpy_revdb_t dinfo;
     disable_io(&dinfo);
-    invoke_after_forward = NULL;
-    invoke_argument = NULL;
     if (setjmp(jmp_buf_cancel_execution) == 0)
-        func(arg);
+        func(cmd, extra);
     enable_io(&dinfo);
 }
 
-struct action_s {
-    const char *name;
-    void (*act)(char *);
-};
-
-static void process_input(char *input, const char *kind, int rpycmd,
-                          struct action_s actions[])
-{
-    char *p;
-    struct action_s *a;
-
-    while (isspace(*input))
-        input++;
-    p = input;
-    while (*p != 0 && !isspace(*p))
-        p++;
-    if (*p != 0) {
-        *p = 0;
-        do {
-            p++;
-        } while (isspace(*p));
-    }
-
-    if (rpycmd) {
-        long i;
-        for (i = 0; rpy_revdb_command_names[i] != NULL; i++) {
-            if (strcmp(rpy_revdb_command_names[i], input) == 0) {
-                execute_rpy_command(i, p);
-                return;
-            }
-        }
-    }
-
-    for (a = actions; a->name != NULL; a++) {
-        if (strcmp(a->name, input) == 0) {
-            a->act(p);
-            return;
-        }
-    }
-    if (strcmp(input, "help") == 0) {
-        printf("select %s:\n", kind);
-        if (rpycmd) {
-            char **p;
-            for (p = rpy_revdb_command_names; *p != NULL; p++)
-                printf("\t%s\n", *p);
-        }
-        for (a = actions; a->name != NULL; a++) {
-            if (*a->name != 0)
-                printf("\t%s\n", a->name);
-        }
-    }
-    else {
-        printf("bad %s '%s', try 'help'\n", kind, input);
-    }
-}
-
-static int read_pipe(int fd, void *buf, ssize_t count)
-{
-    while (count > 0) {
-        ssize_t got = read(fd, buf, count);
-        if (got <= 0)
-            return -1;
-        count -= got;
-        buf += got;
-    }
-    return 0;
-}
-
-static int write_pipe(int fd, const void *buf, ssize_t count)
-{
-    while (count > 0) {
-        ssize_t wrote = write(fd, buf, count);
-        if (wrote <= 0)
-            return -1;
-        count -= wrote;
-        buf += wrote;
-    }
-    return 0;
-}
-
-static int copy_pipe(int dst_fd, int src_fd, ssize_t count)
-{
-    char buffer[16384];
-    while (count > 0) {
-        ssize_t count1 = count > sizeof(buffer) ? sizeof(buffer) : count;
-        if (read_pipe(src_fd, buffer, count1) < 0 ||
-            write_pipe(dst_fd, buffer, count1) < 0)
-            return -1;
-        count -= count1;
-    }
-    return 0;
-}
-
-static void cmd_go(uint64_t target_time, void callback(RPyString *),
-                   RPyString *arg, char mode)
-{
-    struct jump_in_time_s header;
-
-    header.target_time = target_time;
-    header.mode = mode;
-    header.callback = callback;    /* may be NULL */
-    /* ^^^ assumes the fn address is the same in the various forks */
-    header.arg_length = arg == NULL ? 0 : RPyString_Size(arg);
-
-    assert(process_kind == PK_DEBUG_PROCESS);
-    write_pipe(frozen_pipe_signal[WR_SIDE], &header, sizeof(header));
-    if (header.arg_length > 0) {
-        write_pipe(frozen_pipe_signal[WR_SIDE], _RPyString_AsString(arg),
-                   header.arg_length);
-    }
-    exit(0);
-}
-
 static void check_at_end(uint64_t stop_points)
 {
     char dummy[1];
-    struct jump_in_time_s jump_in_time;
-
-    if (process_kind == PK_DEBUG_PROCESS) {
-        printf("At end.\n");
-        cmd_go(stop_points, NULL, NULL, 'g');
-        abort();   /* unreachable */
-    }
-
-    if (process_kind != PK_MAIN_PROCESS) {
-        fprintf(stderr, "[%d] Unexpectedly falling off the end\n",
-                process_kind);
-        exit(1);
-    }
     if (stop_points != rpy_revdb.stop_point_seen) {
         fprintf(stderr, "Bad number of stop points "
-                "(seen %llu, recorded %llu)\n",
-                (unsigned long long)rpy_revdb.stop_point_seen,
-                (unsigned long long)stop_points);
+                "(seen %lld, recorded %lld)\n",
+                (long long)rpy_revdb.stop_point_seen,
+                (long long)stop_points);
         exit(1);
     }
     if (rpy_revdb.buf_p != rpy_revdb.buf_limit ||
@@ -582,134 +461,21 @@
         fprintf(stderr, "RevDB file modified while reading?\n");
         exit(1);
     }
-    if (frozen_num_pipes == 0) {
-        fprintf(stderr, "RevDB file does not contain any stop points\n");
-        exit(1);
-    }
 
-    fprintf(stderr, "\n");
-    fflush(stderr);
-    printf("Replaying finished\n");
-    printf("stop_points=%lld\n", (long long)stop_points);
-
-    close(frozen_pipe_signal[WR_SIDE]);
-    frozen_pipe_signal[WR_SIDE] = -1;
-
-    memset(&jump_in_time, 0, sizeof(jump_in_time));
-    jump_in_time.target_time = frozen_time[frozen_num_pipes-1];
-
-    while (jump_in_time.target_time != (uint64_t)-1) {
-        int p = frozen_num_pipes - 1;
-        if (jump_in_time.target_time > frozen_time[p])
-            jump_in_time.target_time = frozen_time[p];
-        while (frozen_time[p] > jump_in_time.target_time)
-            p--;
-        if (write_pipe(frozen_pipes[p][WR_SIDE],
-                       &jump_in_time, sizeof(jump_in_time)) < 0 ||
-            copy_pipe(frozen_pipes[p][WR_SIDE],
-                      frozen_pipe_signal[RD_SIDE],
-                      jump_in_time.arg_length) < 0) {
-            fprintf(stderr, "broken pipe to frozen subprocess\n");
-            exit(1);
-        }
-        /* blocking here while the p'th frozen process spawns a debug process
-           and the user interacts with it; then: */
-        if (read_pipe(frozen_pipe_signal[RD_SIDE], &jump_in_time,
-                      sizeof(jump_in_time)) < 0) {
-            fprintf(stderr, "broken signal pipe\n");
-            exit(1);
-        }
-    }
+    write_answer(ANSWER_AT_END, 0, 0, 0);
     exit(0);
 }
 
-static void run_frozen_process(int frozen_pipe_fd)
+static void command_fork(void)
 {
-    struct jump_in_time_s jump_in_time;
-    pid_t child_pid;
+    int child_pipes[2];
+    int child_pid;
+    off_t rev_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR);
 
-    while (1) {
-        if (read_pipe(frozen_pipe_fd, &jump_in_time, sizeof(jump_in_time)) < 0)
-            exit(1);
-
-        child_pid = fork();
-        if (child_pid == -1) {
-            perror("fork");
-            exit(1);
-        }
-        if (child_pid == 0) {
-            /* in the child: this is a debug process */
-            process_kind = PK_DEBUG_PROCESS;
-            assert(jump_in_time.target_time >= rpy_revdb.stop_point_seen);
-            most_recent_fork = rpy_revdb.stop_point_seen;
-            switch (jump_in_time.mode) {
-            case 'b':    /* go non-exact: stay at most_recent_fork */
-                rpy_revdb.stop_point_break = most_recent_fork;
-                break;
-            default:     /* other modes: go exact */
-                rpy_revdb.stop_point_break = jump_in_time.target_time;
-            }
-
-            if (jump_in_time.callback == NULL) {
-                assert(jump_in_time.arg_length == 0);
-                assert(invoke_after_forward == NULL);
-            }
-            else {
-                RPyString *s = make_rpy_string(jump_in_time.arg_length);
-                if (read_pipe(frozen_pipe_fd, _RPyString_AsString(s),
-                              jump_in_time.arg_length) < 0) {
-                    fprintf(stderr, "broken pipe to debug subprocess\n");
-                    exit(1);
-                }
-                invoke_after_forward = jump_in_time.callback;
-                invoke_argument = s;
-            }
-            /* continue "running" the RPython program until we reach
-               exactly the specified target_time */
-            break;
-        }
-        else {
-            /* in the parent: the frozen process, which waits for
-               the debug process to finish to reclaim the pid,
-               and then loops to wait for the next wake-up */
-            int status;
-            waitpid(child_pid, &status, 0);
-            if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
-                ;     /* normal exit */
-            else {
-                fprintf(stderr, "debugging subprocess died\n");
-                cmd_go((uint64_t)-1, NULL, NULL, 'q');
-                abort();    /* unreachable */
-            }
-        }
-    }
-}
-
-static void make_new_frozen_process(void)
-{
-    pid_t child_pid;
-    int *fds;
-    off_t fileno_offset;
-
-    if (frozen_num_pipes >= NUM_FROZEN_PROCESSES) {
-        fprintf(stderr, "stop_point_break overflow?\n");
+    if (ancil_recv_fds(RDB_STDIN, child_pipes, 2) != 2) {
+        fprintf(stderr, "cannot read child stdin/stdout pipes\n");
         exit(1);
     }
-    if (frozen_num_pipes == 0)
-        first_created_uid = rpy_revdb.unique_id_seen;
-
-    fprintf(stderr, "[%llu]",
-            (unsigned long long)rpy_revdb.stop_point_seen);
-
-    fds = frozen_pipes[frozen_num_pipes];
-    if (pipe(fds) < 0) {
-        perror("pipe");
-        exit(1);
-    }
-    frozen_time[frozen_num_pipes] = rpy_revdb.stop_point_seen;
-    frozen_num_pipes += 1;
-
-    fileno_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR);
 
     child_pid = fork();
     if (child_pid == -1) {
@@ -717,97 +483,113 @@
         exit(1);
     }
     if (child_pid == 0) {
-        /* in the child: this is a frozen process */
-        process_kind = PK_FROZEN_PROCESS;
-        close(fds[WR_SIDE]);
-        fds[WR_SIDE] = -1;
-        run_frozen_process(fds[RD_SIDE]);
-        /* when we reach that point, we are in the debugging process */
-        lseek(rpy_rev_fileno, fileno_offset, SEEK_SET);
+        /* in the child */
+        if (dup2(child_pipes[0], RDB_STDIN) < 0 ||
+            dup2(child_pipes[1], RDB_STDOUT) < 0) {
+            perror("dup2");
+            exit(1);
+        }
+        /* Close and re-open the revdb log file in the child process.
+           This is the simplest way I found to give 'rpy_rev_fileno'
+           its own offset, independent from the parent.  It assumes
+           that the revdb log file is still the same.  So for Linux,
+           we try to open "/proc/self/fd/%d" instead. */
+        char fd_filename[48];
+        struct stat st;
+        const char *filename;
+        int old_fd = rpy_rev_fileno;
+
+        sprintf(fd_filename, "/proc/self/fd/%d", old_fd);
+        if (lstat(fd_filename, &st) == 0)
+            filename = fd_filename;
+        else
+            filename = rpy_rev_filename;
+        reopen_revdb_file(filename);
+
+        if (close(old_fd) < 0) {
+            perror("close");
+            exit(1);
+        }
+        if (lseek(rpy_rev_fileno, rev_offset, SEEK_SET) < 0) {
+            perror("lseek");
+            exit(1);
+        }
     }
     else {
-        /* in the main process: continue reloading the revdb log */
-        uint64_t remaining = total_stop_points - rpy_revdb.stop_point_break;
-        uint64_t delta;
-        double step = STEP_RATIO;
-        int remaining_freezes = NUM_FROZEN_PROCESSES - frozen_num_pipes;
-        if (step * remaining_freezes < 1.0)
-            step = 1.0 / remaining_freezes;
-        delta = (uint64_t)(remaining * step);
-        if (delta == 0 || delta > remaining || remaining_freezes == 1)
-            rpy_revdb.stop_point_break = total_stop_points;
-        else
-            rpy_revdb.stop_point_break += delta;
-        close(fds[RD_SIDE]);
-        fds[RD_SIDE] = -1;
+        /* in the parent */
+        write_answer(ANSWER_FORKED, child_pid, 0, 0);
+    }
+    /* common code in the parent and the child */
+    if (close(child_pipes[0]) < 0 ||
+        close(child_pipes[1]) < 0) {
+        perror("close");
+        exit(1);
     }
 }
 
-static void act_quit(char *p)
+static void command_forward(rpy_revdb_command_t *cmd)
 {
-    cmd_go((uint64_t)-1, NULL, NULL, 'q');
+    if (cmd->arg1 < rpy_revdb.stop_point_seen) {
+        fprintf(stderr, "CMD_FORWARD: target time %lld < current time %lld\n",
+                (long long)cmd->arg1, (long long)rpy_revdb.stop_point_seen);
+        exit(1);
+    }
+    rpy_revdb.stop_point_break = cmd->arg1;
+    pending_after_forward = &answer_std;
 }
 
-static void act_go(char *p)
+static void command_default(rpy_revdb_command_t *cmd, char *extra)
 {
-    int64_t target_time = strtoll(p, NULL, 10);
-    if (target_time <= 0) {
-        printf("usage: go \n");
-        return;
+    int i;
+    for (i = 0; rpy_revdb_command_names[i] != cmd->cmd; i++) {
+        if (rpy_revdb_command_names[i] == 0) {
+            fprintf(stderr, "unknown command %d\n", cmd->cmd);
+            exit(1);
+        }
     }
-    cmd_go(target_time, NULL, NULL, 'g');
+    execute_rpy_function(rpy_revdb_command_funcs[i], cmd, extra);
 }
 
-static void act_info(char *p)
+RPY_EXTERN
+void rpy_reverse_db_stop_point(void)
 {
-    char cmd = *p;
-    if (cmd == 0)
-        cmd = '?';
-    printf("info %c=%lld\n", cmd, (long long)rpy_reverse_db_get_value(cmd));
-}
-
-static void act_nop(char *p)
-{
-}
-
-static void act_forward(char *p)
-{
-    int64_t delta = strtoll(p, NULL, 10);
-    if (delta <= 0) {
-        if (delta < 0 || *p == 0)
-            printf("usage: forward \n");
-        return;
-    }
-    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + delta;
-}
-
-static void run_debug_process(void)
-{
-    static struct action_s actions_1[] = {
-        { "info", act_info },
-        { "quit", act_quit },
-        { "__go", act_go },
-        { "__forward", act_forward },
-        { "", act_nop },
-        { NULL }
-    };
     while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
         stopped_time = rpy_revdb.stop_point_seen;
         stopped_uid = rpy_revdb.unique_id_seen;
         rpy_revdb.unique_id_seen = (-1ULL) << 63;
-        if (invoke_after_forward != NULL) {
-            execute_rpy_function(invoke_after_forward, invoke_argument);
+
+        if (pending_after_forward) {
+            void (*fn)(void) = pending_after_forward;
+            pending_after_forward = NULL;
+            fn();
         }
         else {
-            char input[256];
-            printf("(%llu)$ ", (unsigned long long)stopped_time);
-            fflush(stdout);
-            if (fgets(input, sizeof(input), stdin) != input) {
-                fprintf(stderr, "\n");
-                act_quit("");
-                abort();   /* unreachable */
+            rpy_revdb_command_t cmd;
+            read_pipe(RDB_STDIN, &cmd, sizeof(cmd));
+
+            char extra[cmd.extra_size + 1];
+            extra[cmd.extra_size] = 0;
+            if (cmd.extra_size > 0)
+                read_pipe(RDB_STDIN, extra, cmd.extra_size);
+
+            switch (cmd.cmd) {
+
+            case CMD_FORK:
+                command_fork();
+                break;
+
+            case CMD_QUIT:
+                exit(0);
+                break;
+
+            case CMD_FORWARD:
+                command_forward(&cmd);
+                break;
+
+            default:
+                command_default(&cmd, extra);
+                break;
             }
-            process_input(input, "command", 1, actions_1);
         }
         rpy_revdb.unique_id_seen = stopped_uid;
         stopped_time = 0;
@@ -816,30 +598,28 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_stop_point(void)
+void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
+                                int64_t arg3, RPyString *extra)
 {
-    if (process_kind == PK_MAIN_PROCESS) {
-        make_new_frozen_process();
-        if (process_kind == PK_MAIN_PROCESS)
-            return;
-    }
-    assert(process_kind == PK_DEBUG_PROCESS);
-    run_debug_process();
-}
-
-RPY_EXTERN
-void rpy_reverse_db_send_output(RPyString *output)
-{
-    fwrite(_RPyString_AsString(output), 1, RPyString_Size(output), stdout);
+    rpy_revdb_command_t c;
+    memset(&c, 0, sizeof(c));
+    c.cmd = cmd;
+    c.extra_size = RPyString_Size(extra);
+    c.arg1 = arg1;
+    c.arg2 = arg2;
+    c.arg3 = arg3;
+    write_pipe(RDB_STDOUT, &c, sizeof(c));
+    if (c.extra_size > 0)
+        write_pipe(RDB_STDOUT, _RPyString_AsString(extra), c.extra_size);
 }
 
 RPY_EXTERN
 void rpy_reverse_db_change_time(char mode, long long time,
-                                void callback(RPyString *), RPyString *arg)
+                                void callback(void))
 {
     switch (mode) {
 
-    case 'f': {      /* forward */
+    case 'f':        /* forward */
         if (time < 0) {
             fprintf(stderr, "revdb.go_forward(): negative amount of steps\n");
             exit(1);
@@ -849,25 +629,22 @@
             exit(1);
         }
         rpy_revdb.stop_point_break = stopped_time + time;
-        invoke_after_forward = callback;
-        invoke_argument = arg;
+        pending_after_forward = callback;
         break;
-    }
-    case 'g':       /* go */
-    case 'b':       /* go non exact */
-        cmd_go(time >= 1 ? time : 1, callback, arg, mode);
-        abort();    /* unreachable */
 
     case 'k':       /* breakpoint */
-        assert(time > 0);
+        if (time <= 0) {
+            fprintf(stderr, "revdb.breakpoint(): non-positive amount of "
+                            "steps\n");
+            exit(1);
+        }
         if (stopped_time != 0) {
             fprintf(stderr, "revdb.breakpoint(): cannot be called from a "
                             "debug command\n");
             exit(1);
         }
         rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + time;
-        invoke_after_forward = callback;
-        invoke_argument = arg;
+        pending_after_forward = callback;
         break;
 
     default:
@@ -881,16 +658,12 @@
     switch (value_id) {
     case 'c':       /* current_time() */
         return stopped_time ? stopped_time : rpy_revdb.stop_point_seen;
-    case 'f':       /* most_recent_fork() */
-        return most_recent_fork;
     case 't':       /* total_time() */
         return total_stop_points;
     case 'b':       /* current_break_time() */
         return rpy_revdb.stop_point_break;
     case 'u':       /* currently_created_objects() */
         return stopped_uid ? stopped_uid : rpy_revdb.unique_id_seen;
-    case '1':       /* first_created_object_uid() */
-        return first_created_uid;
     default:
         return -1;
     }
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -1,5 +1,4 @@
 #include 
-#include 
 
 /* By default, this makes an executable which supports both recording
    and replaying.  It should help avoid troubles like using for
@@ -85,8 +84,8 @@
 #define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
     rpy_reverse_db_send_output(ll_string)
 
-#define OP_REVDB_CHANGE_TIME(mode, time, callback, ll_string, r)        \
-    rpy_reverse_db_change_time(mode, time, callback, ll_string)
+#define OP_REVDB_CHANGE_TIME(mode, time, callback, r)                   \
+    rpy_reverse_db_change_time(mode, time, callback)
 
 #define OP_REVDB_GET_VALUE(value_id, r)                                 \
     r = rpy_reverse_db_get_value(value_id)
@@ -107,8 +106,7 @@
 RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
 RPY_EXTERN void rpy_reverse_db_change_time(char mode, long long time,
-                                           void callback(RPyString *),
-                                           RPyString *arg);
+                                           void callback(void));
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
 RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id,
diff --git a/rpython/translator/revdb/src-revdb/revdb_preinclude.h b/rpython/translator/revdb/src-revdb/revdb_preinclude.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/src-revdb/revdb_preinclude.h
@@ -0,0 +1,9 @@
+#include 
+
+typedef struct rpy_revdb_command_s {
+    int cmd;      /* neg for standard commands, pos for interp-specific */
+    size_t extra_size;
+    int64_t arg1;
+    int64_t arg2;
+    int64_t arg3;
+} rpy_revdb_command_t;

From pypy.commits at gmail.com  Fri Jun 17 15:56:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 12:56:59 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: First I/O test passes again
Message-ID: <5764560b.692dc20a.489bc.ffff9211@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85218:9d04b62f7fde
Date: 2016-06-17 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/9d04b62f7fde/

Log:	First I/O test passes again

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -569,9 +569,7 @@
     'revdb_send_answer':    LLOp(),
     'revdb_change_time':    LLOp(),
     'revdb_get_value':      LLOp(sideeffects=False),
-    ## 'revdb_set_value':      LLOp(),
-    ## 'revdb_identityhash':   LLOp(),
-    ## 'revdb_get_unique_id':  LLOp(sideeffects=False),
+    'revdb_get_unique_id':  LLOp(sideeffects=False),
     ## 'revdb_track_object':   LLOp(),
 }
 # ***** Run test_lloperation after changes. *****
diff --git a/rpython/translator/revdb/ancillary.py b/rpython/translator/revdb/ancillary.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/ancillary.py
@@ -0,0 +1,53 @@
+import os, sys
+from rpython.tool.udir import udir
+
+
+def build(tmpdir):
+    import cffi
+    ffibuilder = cffi.FFI()
+
+    ffibuilder.cdef("""
+        int ancil_send_fds(int, const int *, unsigned);
+        int ancil_recv_fds(int, int *, unsigned);
+    """)
+
+    local_dir = os.path.dirname(os.path.abspath(__file__))
+    src_dir = os.path.join(local_dir, 'src-revdb')
+
+    ffibuilder.set_source("_ancillary_cffi", """
+        #include 
+    """, include_dirs=[src_dir],
+         sources=[os.path.join(src_dir, 'fd_send.c'),
+                  os.path.join(src_dir, 'fd_recv.c')])
+
+    ffibuilder.compile(tmpdir=tmpdir, verbose=True)
+
+def import_():
+    tmpdir = str(udir.ensure('ancillary', dir=1))
+    old_sys_path = sys.path[:]
+    sys.path.insert(0, tmpdir)
+    try:
+        import _ancillary_cffi
+    except ImportError:
+        build(tmpdir)
+        import _ancillary_cffi
+    sys.path[:] = old_sys_path
+    return _ancillary_cffi.ffi, _ancillary_cffi.lib
+
+
+def send_fds(pipe_num, fd_list):
+    ffi, lib = import_()
+    if lib.ancil_send_fds(pipe_num, fd_list, len(fd_list)) < 0:
+        raise OSError(ffi.errno, "ancil_send_fds() failed")
+
+def recv_fds(pipe_num, fd_count):
+    ffi, lib = import_()
+    p = ffi.new("int[]", fd_count)
+    result = lib.ancil_recv_fds(pipe_num, p, fd_count)
+    if result < 0:
+        raise OSError(ffi.errno, "ancil_recv_fds() failed")
+    return [p[i] for i in xrange(result)]
+
+
+if __name__ == '__main__':
+    import_()
diff --git a/rpython/translator/revdb/revmsg.py b/rpython/translator/revdb/revmsg.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/revmsg.py
@@ -0,0 +1,62 @@
+import struct
+
+
+INIT_VERSION_NUMBER   = 0xd80100
+
+CMD_FORK     = -1
+CMD_QUIT     = -2
+CMD_FORWARD  = -3
+
+ANSWER_INIT    = -20
+ANSWER_STD     = -21
+ANSWER_FORKED  = -22
+ANSWER_AT_END  = -23
+
+
+class Message(object):
+    def __init__(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
+        self.cmd = cmd
+        self.arg1 = arg1
+        self.arg2 = arg2
+        self.arg3 = arg3
+        self.extra = extra
+
+    def __eq__(self, other):
+        return (self.cmd == other.cmd and
+                self.arg1 == other.arg1 and
+                self.arg2 == other.arg2 and
+                self.arg3 == other.arg3 and
+                self.extra == other.extra)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class ReplayProcess(object):
+    def __init__(self, stdin, stdout):
+        self.stdin = stdin
+        self.stdout = stdout
+
+    def send(self, msg):
+        binary = struct.pack("iLqqq", msg.cmd, len(msg.extra),
+                             msg.arg1, msg.arg2, msg.arg3)
+        self.stdin.write(binary + msg.extra)
+
+    def recv(self):
+        binary = self.stdout.read(struct.calcsize("iLqqq"))
+        cmd, size, arg1, arg2, arg3 = struct.unpack("iLqqq", binary)
+        extra = self.stdout.read(size) if size > 0 else ""
+        return Message(cmd, arg1, arg2, arg3, extra)
+
+    def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
+        msg = self.recv()
+        assert msg.cmd == cmd
+        if arg1 is not Ellipsis:
+            assert msg.arg1 == arg1
+        if arg2 is not Ellipsis:
+            assert msg.arg2 == arg2
+        if arg3 is not Ellipsis:
+            assert msg.arg3 == arg3
+        if extra is not Ellipsis:
+            assert msg.extra == extra
+        return msg
diff --git a/rpython/translator/revdb/src-revdb/fd_send.c b/rpython/translator/revdb/src-revdb/fd_send.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/src-revdb/fd_send.c
@@ -0,0 +1,92 @@
+/***************************************************************************
+ * libancillary - black magic on Unix domain sockets
+ * (C) Nicolas George
+ * fd_send.c - sending file descriptors
+ ***************************************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _XPG4_2 /* Solaris sucks */
+# define _XPG4_2
+#endif
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#if defined(__FreeBSD__)
+# include  /* FreeBSD sucks */
+#endif
+
+#include "ancillary.h"
+
+int
+ancil_send_fds_with_buffer(int sock, const int *fds, unsigned n_fds, void *buffer)
+{
+    struct msghdr msghdr;
+    char nothing = '!';
+    struct iovec nothing_ptr;
+    struct cmsghdr *cmsg;
+    int i;
+
+    nothing_ptr.iov_base = ¬hing;
+    nothing_ptr.iov_len = 1;
+    msghdr.msg_name = NULL;
+    msghdr.msg_namelen = 0;
+    msghdr.msg_iov = ¬hing_ptr;
+    msghdr.msg_iovlen = 1;
+    msghdr.msg_flags = 0;
+    msghdr.msg_control = buffer;
+    msghdr.msg_controllen = sizeof(struct cmsghdr) + sizeof(int) * n_fds;
+    cmsg = CMSG_FIRSTHDR(&msghdr);
+    cmsg->cmsg_len = msghdr.msg_controllen;
+    cmsg->cmsg_level = SOL_SOCKET;
+    cmsg->cmsg_type = SCM_RIGHTS;
+    for(i = 0; i < n_fds; i++)
+	((int *)CMSG_DATA(cmsg))[i] = fds[i];
+    return(sendmsg(sock, &msghdr, 0) >= 0 ? 0 : -1);
+}
+
+#ifndef SPARE_SEND_FDS
+int
+ancil_send_fds(int sock, const int *fds, unsigned n_fds)
+{
+    ANCIL_FD_BUFFER(ANCIL_MAX_N_FDS) buffer;
+
+    assert(n_fds <= ANCIL_MAX_N_FDS);
+    return(ancil_send_fds_with_buffer(sock, fds, n_fds, &buffer));
+}
+#endif /* SPARE_SEND_FDS */
+
+#ifndef SPARE_SEND_FD
+int
+ancil_send_fd(int sock, int fd)
+{
+    ANCIL_FD_BUFFER(1) buffer;
+
+    return(ancil_send_fds_with_buffer(sock, &fd, 1, &buffer));
+}
+#endif /* SPARE_SEND_FD */
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -529,12 +529,11 @@
 
 static void command_forward(rpy_revdb_command_t *cmd)
 {
-    if (cmd->arg1 < rpy_revdb.stop_point_seen) {
-        fprintf(stderr, "CMD_FORWARD: target time %lld < current time %lld\n",
-                (long long)cmd->arg1, (long long)rpy_revdb.stop_point_seen);
+    if (cmd->arg1 < 0) {
+        fprintf(stderr, "CMD_FORWARD: negative step\n");
         exit(1);
     }
-    rpy_revdb.stop_point_break = cmd->arg1;
+    rpy_revdb.stop_point_break = stopped_time + cmd->arg1;
     pending_after_forward = &answer_std;
 }
 
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -1,5 +1,5 @@
 import py
-import os, sys
+import os, sys, subprocess
 import re, array, struct
 from rpython.tool.udir import udir
 from rpython.translator.interactive import Translation
@@ -8,11 +8,8 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.rtyper.lltypesystem import lltype, llmemory
-"""
-These tests require pexpect (UNIX-only).
-http://pexpect.sourceforge.net/
-"""
-import pexpect
+
+from rpython.translator.revdb.revmsg import *
 
 
 class RDB(object):
@@ -211,19 +208,18 @@
             assert rdb.done()
 
 
+
+
+
+
 class InteractiveTests(object):
-    EOF = pexpect.EOF
 
-    def replay(self, **kwds):
-        kwds.setdefault('timeout', 10)
-        child = pexpect.spawn(str(self.exename),
-                              ['--revdb-replay', str(self.rdbname)], **kwds)
-        child.logfile = sys.stdout
-        def expectx(s):
-            child.expect(re.escape(s))
-        assert not hasattr(child, 'expectx')
-        child.expectx = expectx
-        return child
+    def replay(self):
+        child = subprocess.Popen(
+            [str(self.exename), '--revdb-replay', str(self.rdbname)],
+            stdin  = subprocess.PIPE,
+            stdout = subprocess.PIPE)
+        return ReplayProcess(child.stdin, child.stdout)
 
 
 class TestSimpleInterpreter(InteractiveTests):
@@ -246,14 +242,12 @@
 
     def test_go(self):
         child = self.replay()
-        child.expectx('stop_points=3\r\n'
-                      '(3)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('')
-        child.expectx('(1)$ ')
-        child.sendline('__go 52')
-        child.expectx('(3)$ ')
+        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
+        child.expect(ANSWER_STD, 1, Ellipsis)
+        child.send(Message(CMD_FORWARD, 2))
+        child.expect(ANSWER_STD, 3, Ellipsis)
+        child.send(Message(CMD_FORWARD, 2))
+        child.expect(ANSWER_AT_END)
 
     def test_help(self):
         child = self.replay()

From pypy.commits at gmail.com  Fri Jun 17 16:07:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 13:07:29 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix/kill tests
Message-ID: <57645881.e7c9c20a.9dfba.ffff8f4a@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85219:0a8242113d71
Date: 2016-06-17 22:08 +0200
http://bitbucket.org/pypy/pypy/changeset/0a8242113d71/

Log:	Fix/kill tests

diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -208,18 +208,15 @@
             assert rdb.done()
 
 
-
-
-
-
 class InteractiveTests(object):
 
     def replay(self):
-        child = subprocess.Popen(
+        subproc = subprocess.Popen(
             [str(self.exename), '--revdb-replay', str(self.rdbname)],
             stdin  = subprocess.PIPE,
             stdout = subprocess.PIPE)
-        return ReplayProcess(child.stdin, child.stdout)
+        self.subproc = subproc
+        return ReplayProcess(subproc.stdin, subproc.stdout)
 
 
 class TestSimpleInterpreter(InteractiveTests):
@@ -249,40 +246,12 @@
         child.send(Message(CMD_FORWARD, 2))
         child.expect(ANSWER_AT_END)
 
-    def test_help(self):
-        child = self.replay()
-        child.sendline('help')
-        child.expectx('select command:\r\n')
-        # ...
-        child.expectx('(3)$ ')
-        child.sendline('info')
-        child.expectx("info ?=-1\r\n")
-
-    def test_info_fork(self):
-        child = self.replay()
-        child.sendline('info fork')
-        child.expectx('info f=3\r\n')
-
     def test_quit(self):
         child = self.replay()
-        child.sendline('quit')
-        child.expect(self.EOF)
-
-    def test_forward(self):
-        child = self.replay()
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('__forward 1')
-        child.expectx('(2)$ ')
-        child.sendline('__forward 1')
-        child.expectx('(3)$ ')
-        child.sendline('info fork')
-        child.expectx('info f=1\r\n')
-        child.sendline('__forward 1')
-        child.expectx('At end.\r\n'
-                      '(3)$ ')
-        child.sendline('info fork')
-        child.expectx('info f=3\r\n')
+        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
+        child.expect(ANSWER_STD, 1, Ellipsis)
+        child.send(Message(CMD_QUIT))
+        assert self.subproc.wait() == 0
 
 
 class TestDebugCommands(InteractiveTests):

From pypy.commits at gmail.com  Sat Jun 18 01:49:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 22:49:58 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix more tests
Message-ID: <5764e106.05b01c0a.32686.ffff9c31@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85221:6dd8ff533d6d
Date: 2016-06-18 07:46 +0200
http://bitbucket.org/pypy/pypy/changeset/6dd8ff533d6d/

Log:	fix more tests

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -103,7 +103,8 @@
                           ('cmd', rffi.INT),
                           ('arg1', lltype.SignedLongLong),
                           ('arg2', lltype.SignedLongLong),
-                          ('arg3', lltype.SignedLongLong))
+                          ('arg3', lltype.SignedLongLong),
+                          hints={'ignore_revdb': True})
 
 
 class RegisterDebugCommand(ExtRegistryEntry):
@@ -126,9 +127,9 @@
             cmds.append((command_num, func))
             s_func = self.bookkeeper.immutablevalue(func)
             s_ptr1 = llannotation.SomePtr(ll_ptrtype=_CMDPTR)
-            s_ptr2 = llannotation.SomePtr(ll_ptrtype=rffi.CCHARP)
+            s_str2 = annmodel.SomeString()
             self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                             s_func, [s_ptr1, s_ptr2])
+                                             s_func, [s_ptr1, s_str2])
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -439,7 +439,8 @@
         if self.db.reverse_debugger:
             S = self.lltypemap(op.args[0]).TO
             if (S._gckind != 'gc' and not S._hints.get('is_excdata')
-                    and not S._hints.get('static_immutable')):
+                    and not S._hints.get('static_immutable')
+                    and not S._hints.get('ignore_revdb')):
                 from rpython.translator.revdb import revdb_genc
                 result = revdb_genc.emit(result, self.lltypename(op.result),
                                          newvalue)
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/revdb_genc.py
--- a/rpython/translator/revdb/revdb_genc.py
+++ b/rpython/translator/revdb/revdb_genc.py
@@ -24,7 +24,7 @@
 
 def prepare_database(db):
     FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR,
-                                          rffi.CCHARP], lltype.Void))
+                                          lltype.Ptr(rstr.STR)], lltype.Void))
 
     bk = db.translator.annotator.bookkeeper
     cmds = getattr(db.translator, 'revdb_commands', [])
diff --git a/rpython/translator/revdb/revmsg.py b/rpython/translator/revdb/revmsg.py
--- a/rpython/translator/revdb/revmsg.py
+++ b/rpython/translator/revdb/revmsg.py
@@ -22,6 +22,11 @@
         self.arg3 = arg3
         self.extra = extra
 
+    def __repr__(self):
+        return 'Message(%d, %d, %d, %d, %r)' % (self.cmd, self.arg1,
+                                                self.arg2, self.arg3,
+                                                self.extra)
+
     def __eq__(self, other):
         return (self.cmd == other.cmd and
                 self.arg1 == other.arg1 and
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -201,7 +201,7 @@
 #define ANSWER_FORKED  (-22)
 #define ANSWER_AT_END  (-23)
 
-typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, char *);
+typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, RPyString *);
 
 static int rpy_rev_sockfd;
 static const char *rpy_rev_filename;
@@ -210,6 +210,7 @@
 static uint64_t total_stop_points;
 static jmp_buf jmp_buf_cancel_execution;
 static void (*pending_after_forward)(void);
+static RPyString *empty_string;
 
 static void attach_gdb(void)
 {
@@ -285,6 +286,20 @@
                  rpy_revdb.unique_id_seen, 0);
 }
 
+static RPyString *make_rpy_string(size_t length)
+{
+    RPyString *s = malloc(sizeof(RPyString) + length);
+    if (s == NULL) {
+        fprintf(stderr, "out of memory for a string of %llu chars\n",
+                (unsigned long long)length);
+        exit(1);
+    }
+    /* xxx assumes Boehm here for now */
+    memset(s, 0, sizeof(RPyString));
+    RPyString_Size(s) = length;
+    return s;
+}
+
 static void reopen_revdb_file(const char *filename)
 {
     rpy_rev_fileno = open(filename, O_RDONLY | O_NOCTTY);
@@ -347,6 +362,8 @@
     rpy_revdb.stop_point_break = 1;
     rpy_revdb.unique_id_seen = 1;
 
+    empty_string = make_rpy_string(0);
+
     write_answer(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points, 0);
     pending_after_forward = &answer_std;
 
@@ -374,8 +391,8 @@
            running some custom code now, and we can't just perform I/O
            or access raw memory---because there is no raw memory! 
         */
-        printf("%s:%d: Attempted to do I/O or access raw memory\n",
-               file, line);
+        fprintf(stderr, "%s:%d: Attempted to do I/O or access raw memory\n",
+                file, line);
         longjmp(jmp_buf_cancel_execution, 1);
     }
 }
@@ -413,26 +430,9 @@
     pypy_g_ExcData.ed_exc_value = dinfo->saved_exc[1];
 }
 
-/*
-static void execute_rpy_function(void func(RPyString *), RPyString *arg);
-
-static void execute_rpy_command(long index, char *arguments)
-{
-    size_t length = strlen(arguments);
-    RPyString *s;
-
-    while (length > 0 && isspace(arguments[length - 1]))
-        length--;
-    s = make_rpy_string(length);
-    memcpy(_RPyString_AsString(s), arguments, length);
-
-    execute_rpy_function(rpy_revdb_command_funcs[index], s);
-}
-*/
-
 static void execute_rpy_function(rpy_revdb_command_fn func,
                                  rpy_revdb_command_t *cmd,
-                                 char *extra)
+                                 RPyString *extra)
 {
     rpy_revdb_t dinfo;
     disable_io(&dinfo);
@@ -533,6 +533,7 @@
 
 static void command_default(rpy_revdb_command_t *cmd, char *extra)
 {
+    RPyString *s;
     int i;
     for (i = 0; rpy_revdb_command_names[i] != cmd->cmd; i++) {
         if (rpy_revdb_command_names[i] == 0) {
@@ -540,7 +541,15 @@
             exit(1);
         }
     }
-    execute_rpy_function(rpy_revdb_command_funcs[i], cmd, extra);
+
+    if (cmd->extra_size == 0) {
+        s = empty_string;
+    }
+    else {
+        s = make_rpy_string(cmd->extra_size);
+        memcpy(_RPyString_AsString(s), extra, cmd->extra_size);
+    }
+    execute_rpy_function(rpy_revdb_command_funcs[i], cmd, s);
 }
 
 RPY_EXTERN
@@ -688,11 +697,11 @@
         return;
     }
     if (unique_id <= 0) {
-        printf("cannot track a prebuilt or debugger-created object\n");
+        fprintf(stderr, "cannot track a prebuilt or debugger-created object\n");
         return;
     }
     if (unique_id < stopped_uid) {
-        printf("cannot track the creation of an object already created\n");
+        fprintf(stderr, "cannot track the creation of an object already created\n");
         return;
     }
     assert(callback != NULL);
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -81,8 +81,8 @@
     if (++rpy_revdb.stop_point_seen == rpy_revdb.stop_point_break)      \
         rpy_reverse_db_stop_point()
 
-#define OP_REVDB_SEND_OUTPUT(ll_string, r)                              \
-    rpy_reverse_db_send_output(ll_string)
+#define OP_REVDB_SEND_ANSWER(cmd, arg1, arg2, arg3, ll_string, r)       \
+    rpy_reverse_db_send_answer(cmd, arg1, arg2, arg3, ll_string)
 
 #define OP_REVDB_CHANGE_TIME(mode, time, callback, r)                   \
     rpy_reverse_db_change_time(mode, time, callback)
@@ -103,7 +103,8 @@
 RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size,
                                       const char *file, int line);
 RPY_EXTERN void rpy_reverse_db_stop_point(void);
-RPY_EXTERN void rpy_reverse_db_send_output(RPyString *output);
+RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
+                                           int64_t arg3, RPyString *extra);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
 RPY_EXTERN void rpy_reverse_db_change_time(char mode, long long time,
                                            void callback(void));
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -3,9 +3,9 @@
 import re, array, struct
 from rpython.tool.udir import udir
 from rpython.translator.interactive import Translation
-from rpython.rlib.rarithmetic import LONG_BIT
+from rpython.rlib.rarithmetic import LONG_BIT, intmask
 from rpython.rlib import objectmodel, revdb
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.debug import debug_print
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.rtyper.lltypesystem import lltype, llmemory
 
@@ -210,14 +210,18 @@
 
 class InteractiveTests(object):
 
-    def replay(self):
+    def replay(self, **kwds):
         s1, s2 = socket.socketpair()
         subproc = subprocess.Popen(
             [str(self.exename), '--revdb-replay', str(self.rdbname),
-             str(s2.fileno())])
+             str(s2.fileno())], **kwds)
         s2.close()
         self.subproc = subproc
-        return ReplayProcess(subproc.pid, s1)
+        child = ReplayProcess(subproc.pid, s1)
+        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
+        msg = child.expect(ANSWER_STD, 1, Ellipsis)
+        self.total_stop_points = msg.arg2
+        return child
 
 
 class TestSimpleInterpreter(InteractiveTests):
@@ -240,8 +244,6 @@
 
     def test_go(self):
         child = self.replay()
-        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
-        child.expect(ANSWER_STD, 1, Ellipsis)
         child.send(Message(CMD_FORWARD, 2))
         child.expect(ANSWER_STD, 3, Ellipsis)
         child.send(Message(CMD_FORWARD, 2))
@@ -249,15 +251,11 @@
 
     def test_quit(self):
         child = self.replay()
-        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
-        child.expect(ANSWER_STD, 1, Ellipsis)
         child.send(Message(CMD_QUIT))
         assert self.subproc.wait() == 0
 
     def test_fork(self):
         child = self.replay()
-        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
-        child.expect(ANSWER_STD, 1, Ellipsis)
         child2 = child.fork()
         child.send(Message(CMD_FORWARD, 2))
         child.expect(ANSWER_STD, 3, Ellipsis)
@@ -298,49 +296,50 @@
             revdb.send_output("callback_track_obj\n")
             dbstate.gcref = gcref
         #
-        def blip(cmdline):
-            revdb.send_output('<<<' + cmdline + '>>>\n')
-            if cmdline == 'oops':
+        def blip(cmd, extra):
+            debug_print('<<<', cmd.c_cmd, cmd.c_arg1,
+                               cmd.c_arg2, cmd.c_arg3, extra, '>>>')
+            if extra == 'oops':
                 for i in range(1000):
                     print 42     # I/O not permitted
-            if cmdline == 'raise-and-catch':
-                try:
-                    g(cmdline)
-                except ValueError:
-                    pass
-            if cmdline == 'crash':
-                raise ValueError
-            if cmdline == 'get-value':
-                revdb.send_output('%d,%d,%d\n' % (revdb.current_time(),
-                                                  revdb.most_recent_fork(),
-                                                  revdb.total_time()))
-            if cmdline == 'go-fw':
-                revdb.go_forward(1, went_fw, "xx")
-            if cmdline == 'change-time':
-                revdb.jump_in_time(2, changed_time, "xyzzy")
-            if cmdline == 'change-time-non-exact':
-                revdb.jump_in_time(2, changed_time, "nonx", exact=False)
-            if cmdline == 'set-break-after-0':
-                dbstate.break_after = 0
-            if cmdline == 'print-id':
-                revdb.send_output('obj.x=%d %d %d\n' % (
-                    dbstate.stuff.x,
-                    revdb.get_unique_id(dbstate.stuff),
-                    revdb.currently_created_objects()))
-            if cmdline.startswith('track-object '):
-                uid = int(cmdline[len('track-object '):])
-                dbstate.gcref = lltype.nullptr(llmemory.GCREF.TO)
-                revdb.track_object(uid, callback_track_obj)
-            if cmdline == 'get-tracked-object':
-                if dbstate.gcref:
-                    revdb.send_output('got obj.x=%d\n' % (
-                        cast_gcref_to_instance(Stuff, dbstate.gcref).x,))
-                else:
-                    revdb.send_output('none\n')
-            if cmdline == 'first-created-uid':
-                revdb.send_output('first-created-uid=%d\n' % (
-                    revdb.first_created_object_uid(),))
-            revdb.send_output('blipped\n')
+            ## if cmdline == 'raise-and-catch':
+            ##     try:
+            ##         g(cmdline)
+            ##     except ValueError:
+            ##         pass
+            ## if cmdline == 'crash':
+            ##     raise ValueError
+            ## if cmdline == 'get-value':
+            ##     revdb.send_output('%d,%d,%d\n' % (revdb.current_time(),
+            ##                                       revdb.most_recent_fork(),
+            ##                                       revdb.total_time()))
+            ## if cmdline == 'go-fw':
+            ##     revdb.go_forward(1, went_fw, "xx")
+            ## if cmdline == 'change-time':
+            ##     revdb.jump_in_time(2, changed_time, "xyzzy")
+            ## if cmdline == 'change-time-non-exact':
+            ##     revdb.jump_in_time(2, changed_time, "nonx", exact=False)
+            ## if cmdline == 'set-break-after-0':
+            ##     dbstate.break_after = 0
+            ## if cmdline == 'print-id':
+            ##     revdb.send_output('obj.x=%d %d %d\n' % (
+            ##         dbstate.stuff.x,
+            ##         revdb.get_unique_id(dbstate.stuff),
+            ##         revdb.currently_created_objects()))
+            ## if cmdline.startswith('track-object '):
+            ##     uid = int(cmdline[len('track-object '):])
+            ##     dbstate.gcref = lltype.nullptr(llmemory.GCREF.TO)
+            ##     revdb.track_object(uid, callback_track_obj)
+            ## if cmdline == 'get-tracked-object':
+            ##     if dbstate.gcref:
+            ##         revdb.send_output('got obj.x=%d\n' % (
+            ##             cast_gcref_to_instance(Stuff, dbstate.gcref).x,))
+            ##     else:
+            ##         revdb.send_output('none\n')
+            ## if cmdline == 'first-created-uid':
+            ##     revdb.send_output('first-created-uid=%d\n' % (
+            ##         revdb.first_created_object_uid(),))
+            revdb.send_answer(42, cmd.c_cmd, -43, -44, "foo")
         lambda_blip = lambda: blip
         #
         class DBState:
@@ -348,14 +347,14 @@
         dbstate = DBState()
         #
         def main(argv):
-            revdb.register_debug_command('r', lambda_blip)
+            revdb.register_debug_command(1, lambda_blip)
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
                 revdb.stop_point()
-                if i == dbstate.break_after:
-                    revdb.send_output('breakpoint!\n')
-                    revdb.go_forward(1, _nothing, "")
+                ## if i == dbstate.break_after:
+                ##     revdb.send_output('breakpoint!\n')
+                ##     revdb.go_forward(1, _nothing, "")
                 print op
             return 9
         compile(cls, main, [], backendopt=False)
@@ -363,19 +362,15 @@
 
     def test_run_blip(self):
         child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r  foo  bar  baz  ')
-        child.expectx('<<>>\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
+        child.send(Message(1, extra=''))
+        child.expect(42, 1, -43, -44, "foo")
 
     def test_io_not_permitted(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r oops')
-        child.expectx('<<>>\r\n')
-        child.expectx(': Attempted to do I/O or access raw memory\r\n'
-                      '(3)$ ')
+        child = self.replay(stderr=subprocess.PIPE)
+        child.send(Message(1, extra='oops'))
+        child.close()
+        line = self.subproc.stderr.read()
+        assert line.endswith(': Attempted to do I/O or access raw memory\n')
 
     def test_interaction_with_forward(self):
         child = self.replay()

From pypy.commits at gmail.com  Sat Jun 18 01:49:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 22:49:56 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fixes, progress
Message-ID: <5764e104.cdcf1c0a.c8f24.ffff9921@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85220:c89a87929045
Date: 2016-06-18 07:22 +0200
http://bitbucket.org/pypy/pypy/changeset/c89a87929045/

Log:	fixes, progress

diff --git a/rpython/translator/revdb/revmsg.py b/rpython/translator/revdb/revmsg.py
--- a/rpython/translator/revdb/revmsg.py
+++ b/rpython/translator/revdb/revmsg.py
@@ -1,4 +1,5 @@
-import struct
+import os, struct, socket, errno
+from rpython.translator.revdb import ancillary
 
 
 INIT_VERSION_NUMBER   = 0xd80100
@@ -33,19 +34,29 @@
 
 
 class ReplayProcess(object):
-    def __init__(self, stdin, stdout):
-        self.stdin = stdin
-        self.stdout = stdout
+    def __init__(self, pid, control_socket):
+        self.pid = pid
+        self.control_socket = control_socket
+
+    def _recv_all(self, size):
+        pieces = []
+        while size > 0:
+            data = self.control_socket.recv(size)
+            if not data:
+                raise EOFError
+            size -= len(data)
+            pieces.append(data)
+        return ''.join(pieces)
 
     def send(self, msg):
-        binary = struct.pack("iLqqq", msg.cmd, len(msg.extra),
+        binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
                              msg.arg1, msg.arg2, msg.arg3)
-        self.stdin.write(binary + msg.extra)
+        self.control_socket.sendall(binary + msg.extra)
 
     def recv(self):
-        binary = self.stdout.read(struct.calcsize("iLqqq"))
-        cmd, size, arg1, arg2, arg3 = struct.unpack("iLqqq", binary)
-        extra = self.stdout.read(size) if size > 0 else ""
+        binary = self._recv_all(struct.calcsize("iIqqq"))
+        cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
+        extra = self._recv_all(size)
         return Message(cmd, arg1, arg2, arg3, extra)
 
     def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
@@ -60,3 +71,15 @@
         if extra is not Ellipsis:
             assert msg.extra == extra
         return msg
+
+    def fork(self):
+        self.send(Message(CMD_FORK))
+        s1, s2 = socket.socketpair()
+        ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
+        s2.close()
+        msg = self.expect(ANSWER_FORKED, Ellipsis)
+        child_pid = msg.arg1
+        return ReplayProcess(child_pid, s1)
+
+    def close(self):
+        self.send(Message(CMD_QUIT))
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -201,11 +201,9 @@
 #define ANSWER_FORKED  (-22)
 #define ANSWER_AT_END  (-23)
 
-#define RDB_STDIN   0
-#define RDB_STDOUT  1
-
 typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, char *);
 
+static int rpy_rev_sockfd;
 static const char *rpy_rev_filename;
 static uint64_t stopped_time;
 static uint64_t stopped_uid;
@@ -244,13 +242,12 @@
     (void)read_at_least(buf, count, count);
 }
 
-
-static void read_pipe(int fd, void *buf, ssize_t count)
+static void read_sock(void *buf, ssize_t count)
 {
     while (count > 0) {
-        ssize_t got = read(fd, buf, count);
+        ssize_t got = read(rpy_rev_sockfd, buf, count);
         if (got <= 0) {
-            fprintf(stderr, "subprocess: cannot read pipe %d\n", fd);
+            fprintf(stderr, "subprocess: cannot read from control socket\n");
             exit(1);
         }
         count -= got;
@@ -258,12 +255,12 @@
     }
 }
 
-static void write_pipe(int fd, const void *buf, ssize_t count)
+static void write_sock(const void *buf, ssize_t count)
 {
     while (count > 0) {
-        ssize_t wrote = write(fd, buf, count);
+        ssize_t wrote = write(rpy_rev_sockfd, buf, count);
         if (wrote <= 0) {
-            fprintf(stderr, "subprocess: cannot write to pipe %d\n", fd);
+            fprintf(stderr, "subprocess: cannot write to control socket\n");
             exit(1);
         }
         count -= wrote;
@@ -279,7 +276,7 @@
     c.arg1 = arg1;
     c.arg2 = arg2;
     c.arg3 = arg3;
-    write_pipe(RDB_STDOUT, &c, sizeof(c));
+    write_sock(&c, sizeof(c));
 }
 
 static void answer_std(void)
@@ -305,12 +302,14 @@
     char input[16];
     ssize_t count;
 
-    if (argc != 3) {
-        fprintf(stderr, "syntax: %s --revdb-replay \n", argv[0]);
+    if (argc != 4) {
+        fprintf(stderr, "syntax: %s --revdb-replay  \n",
+                argv[0]);
         exit(2);
     }
     rpy_rev_filename = argv[2];
     reopen_revdb_file(rpy_rev_filename);
+    rpy_rev_sockfd = atoi(argv[3]);
 
     assert(RPY_RDB_REPLAY == 1);
 
@@ -468,15 +467,14 @@
 
 static void command_fork(void)
 {
-    int child_pipes[2];
+    int child_sockfd;
     int child_pid;
     off_t rev_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR);
 
-    if (ancil_recv_fds(RDB_STDIN, child_pipes, 2) != 2) {
-        fprintf(stderr, "cannot read child stdin/stdout pipes\n");
+    if (ancil_recv_fd(rpy_rev_sockfd, &child_sockfd) < 0) {
+        fprintf(stderr, "cannot fetch child control socket: %m\n");
         exit(1);
     }
-
     child_pid = fork();
     if (child_pid == -1) {
         perror("fork");
@@ -484,11 +482,12 @@
     }
     if (child_pid == 0) {
         /* in the child */
-        if (dup2(child_pipes[0], RDB_STDIN) < 0 ||
-            dup2(child_pipes[1], RDB_STDOUT) < 0) {
-            perror("dup2");
+        if (close(rpy_rev_sockfd) < 0) {
+            perror("close");
             exit(1);
         }
+        rpy_rev_sockfd = child_sockfd;
+
         /* Close and re-open the revdb log file in the child process.
            This is the simplest way I found to give 'rpy_rev_fileno'
            its own offset, independent from the parent.  It assumes
@@ -518,12 +517,7 @@
     else {
         /* in the parent */
         write_answer(ANSWER_FORKED, child_pid, 0, 0);
-    }
-    /* common code in the parent and the child */
-    if (close(child_pipes[0]) < 0 ||
-        close(child_pipes[1]) < 0) {
-        perror("close");
-        exit(1);
+        close(child_sockfd);
     }
 }
 
@@ -564,12 +558,12 @@
         }
         else {
             rpy_revdb_command_t cmd;
-            read_pipe(RDB_STDIN, &cmd, sizeof(cmd));
+            read_sock(&cmd, sizeof(cmd));
 
             char extra[cmd.extra_size + 1];
             extra[cmd.extra_size] = 0;
             if (cmd.extra_size > 0)
-                read_pipe(RDB_STDIN, extra, cmd.extra_size);
+                read_sock(extra, cmd.extra_size);
 
             switch (cmd.cmd) {
 
@@ -601,15 +595,19 @@
                                 int64_t arg3, RPyString *extra)
 {
     rpy_revdb_command_t c;
-    memset(&c, 0, sizeof(c));
+    size_t extra_size = RPyString_Size(extra);
     c.cmd = cmd;
-    c.extra_size = RPyString_Size(extra);
+    c.extra_size = extra_size;
+    if (c.extra_size != extra_size) {
+        fprintf(stderr, "string too large (more than 4GB)\n");
+        exit(1);
+    }
     c.arg1 = arg1;
     c.arg2 = arg2;
     c.arg3 = arg3;
-    write_pipe(RDB_STDOUT, &c, sizeof(c));
-    if (c.extra_size > 0)
-        write_pipe(RDB_STDOUT, _RPyString_AsString(extra), c.extra_size);
+    write_sock(&c, sizeof(c));
+    if (extra_size > 0)
+        write_sock(_RPyString_AsString(extra), extra_size);
 }
 
 RPY_EXTERN
diff --git a/rpython/translator/revdb/src-revdb/revdb_preinclude.h b/rpython/translator/revdb/src-revdb/revdb_preinclude.h
--- a/rpython/translator/revdb/src-revdb/revdb_preinclude.h
+++ b/rpython/translator/revdb/src-revdb/revdb_preinclude.h
@@ -1,8 +1,8 @@
 #include 
 
 typedef struct rpy_revdb_command_s {
-    int cmd;      /* neg for standard commands, pos for interp-specific */
-    size_t extra_size;
+    int32_t cmd;      /* neg for standard commands, pos for interp-specific */
+    uint32_t extra_size;
     int64_t arg1;
     int64_t arg2;
     int64_t arg3;
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -1,5 +1,5 @@
 import py
-import os, sys, subprocess
+import os, sys, subprocess, socket
 import re, array, struct
 from rpython.tool.udir import udir
 from rpython.translator.interactive import Translation
@@ -211,12 +211,13 @@
 class InteractiveTests(object):
 
     def replay(self):
+        s1, s2 = socket.socketpair()
         subproc = subprocess.Popen(
-            [str(self.exename), '--revdb-replay', str(self.rdbname)],
-            stdin  = subprocess.PIPE,
-            stdout = subprocess.PIPE)
+            [str(self.exename), '--revdb-replay', str(self.rdbname),
+             str(s2.fileno())])
+        s2.close()
         self.subproc = subproc
-        return ReplayProcess(subproc.stdin, subproc.stdout)
+        return ReplayProcess(subproc.pid, s1)
 
 
 class TestSimpleInterpreter(InteractiveTests):
@@ -253,6 +254,19 @@
         child.send(Message(CMD_QUIT))
         assert self.subproc.wait() == 0
 
+    def test_fork(self):
+        child = self.replay()
+        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
+        child.expect(ANSWER_STD, 1, Ellipsis)
+        child2 = child.fork()
+        child.send(Message(CMD_FORWARD, 2))
+        child.expect(ANSWER_STD, 3, Ellipsis)
+        child2.send(Message(CMD_FORWARD, 1))
+        child2.expect(ANSWER_STD, 2, Ellipsis)
+        #
+        child.close()
+        child2.close()
+
 
 class TestDebugCommands(InteractiveTests):
 

From pypy.commits at gmail.com  Sat Jun 18 01:50:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 22:50:00 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: next tests
Message-ID: <5764e108.8f1d1c0a.10cf8.ffff9b32@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85222:04a4ccfafb84
Date: 2016-06-18 07:50 +0200
http://bitbucket.org/pypy/pypy/changeset/04a4ccfafb84/

Log:	next tests

diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -302,13 +302,13 @@
             if extra == 'oops':
                 for i in range(1000):
                     print 42     # I/O not permitted
-            ## if cmdline == 'raise-and-catch':
-            ##     try:
-            ##         g(cmdline)
-            ##     except ValueError:
-            ##         pass
-            ## if cmdline == 'crash':
-            ##     raise ValueError
+            if extra == 'raise-and-catch':
+                try:
+                    g(extra)
+                except ValueError:
+                    pass
+            if extra == 'crash':
+                raise ValueError
             ## if cmdline == 'get-value':
             ##     revdb.send_output('%d,%d,%d\n' % (revdb.current_time(),
             ##                                       revdb.most_recent_fork(),
@@ -339,7 +339,7 @@
             ## if cmdline == 'first-created-uid':
             ##     revdb.send_output('first-created-uid=%d\n' % (
             ##         revdb.first_created_object_uid(),))
-            revdb.send_answer(42, cmd.c_cmd, -43, -44, "foo")
+            revdb.send_answer(42, cmd.c_cmd, -43, -44, extra)
         lambda_blip = lambda: blip
         #
         class DBState:
@@ -362,44 +362,33 @@
 
     def test_run_blip(self):
         child = self.replay()
-        child.send(Message(1, extra=''))
-        child.expect(42, 1, -43, -44, "foo")
+        child.send(Message(1, extra='foo'))
+        child.expect(42, 1, -43, -44, 'foo')
 
     def test_io_not_permitted(self):
         child = self.replay(stderr=subprocess.PIPE)
         child.send(Message(1, extra='oops'))
         child.close()
-        line = self.subproc.stderr.read()
-        assert line.endswith(': Attempted to do I/O or access raw memory\n')
+        err = self.subproc.stderr.read()
+        assert err.endswith(': Attempted to do I/O or access raw memory\n')
 
     def test_interaction_with_forward(self):
         child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r oops')
-        child.expectx('<<>>\r\n')
-        child.expectx('Attempted to do I/O or access raw memory\r\n'
-                      '(1)$ ')
-        child.sendline('__forward 50')
-        child.expectx('At end.\r\n'
-                      '(3)$ ')
+        child.send(Message(1, extra='oops'))
+        child.send(Message(CMD_FORWARD, 50))
+        child.expect(ANSWER_AT_END)
 
     def test_raise_and_catch(self):
         child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r raise-and-catch')
-        child.expectx('<<>>\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
+        child.send(Message(1, extra='raise-and-catch'))
+        child.expect(42, 1, -43, -44, 'raise-and-catch')
 
     def test_crash(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r crash')
-        child.expectx('<<>>\r\n'
-                      'Command crashed with ValueError\r\n'
-                      '(3)$ ')
+        child = self.replay(stderr=subprocess.PIPE)
+        child.send(Message(1, extra='crash'))
+        child.close()
+        err = self.subproc.stderr.read()
+        assert err.endswith('Command crashed with ValueError\n')
 
     def test_get_value(self):
         child = self.replay()

From pypy.commits at gmail.com  Sat Jun 18 02:00:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 17 Jun 2016 23:00:38 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: next tests
Message-ID: <5764e386.8aa9c20a.768a.00d1@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85223:17ab8365d4b1
Date: 2016-06-18 08:01 +0200
http://bitbucket.org/pypy/pypy/changeset/17ab8365d4b1/

Log:	next tests

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -86,17 +86,9 @@
 
 @specialize.arg(2)
 def _change_time(mode, time, callback):
-    callback_wrapper = _make_callback(callback)
-    ll_callback = llhelper(_CALLBACK_ARG_FNPTR, callback_wrapper)
+    ll_callback = llhelper(_CALLBACK_NOARG_FNPTR, callback)
     llop.revdb_change_time(lltype.Void, mode, time, ll_callback)
-
- at specialize.memo()
-def _make_callback(callback):
-    def callback_wrapper(ll_string):
-        callback(hlstr(ll_string))
-    return callback_wrapper
-_CALLBACK_ARG_FNPTR = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)],
-                                                 lltype.Void))
+_CALLBACK_NOARG_FNPTR = lltype.Ptr(lltype.FuncType([], lltype.Void))
 _CALLBACK_GCREF_FNPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
                                                    lltype.Void))
 _CMDPTR = rffi.CStructPtr('rpy_revdb_command_s',
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -219,8 +219,7 @@
         self.subproc = subproc
         child = ReplayProcess(subproc.pid, s1)
         child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
-        msg = child.expect(ANSWER_STD, 1, Ellipsis)
-        self.total_stop_points = msg.arg2
+        child.expect(ANSWER_STD, 1, Ellipsis)
         return child
 
 
@@ -278,16 +277,10 @@
                 raise ValueError
         g._dont_inline_ = True
         #
-        def went_fw(arg):
-            revdb.send_output('went-fw %s -> %d\n' % (arg,
-                                                      revdb.current_time()))
+        def went_fw():
+            revdb.send_answer(120, revdb.current_time())
             if revdb.current_time() != revdb.total_time():
-                revdb.go_forward(1, went_fw, "yy")
-        def changed_time(arg):
-            revdb.send_output('changed-time %s -> %d\n' % (arg,
-                                                      revdb.current_time()))
-            if revdb.current_time() != revdb.total_time():
-                revdb.go_forward(1, went_fw, "zz")
+                revdb.go_forward(1, went_fw)
         #
         def _nothing(arg):
             pass
@@ -309,16 +302,11 @@
                     pass
             if extra == 'crash':
                 raise ValueError
-            ## if cmdline == 'get-value':
-            ##     revdb.send_output('%d,%d,%d\n' % (revdb.current_time(),
-            ##                                       revdb.most_recent_fork(),
-            ##                                       revdb.total_time()))
-            ## if cmdline == 'go-fw':
-            ##     revdb.go_forward(1, went_fw, "xx")
-            ## if cmdline == 'change-time':
-            ##     revdb.jump_in_time(2, changed_time, "xyzzy")
-            ## if cmdline == 'change-time-non-exact':
-            ##     revdb.jump_in_time(2, changed_time, "nonx", exact=False)
+            if extra == 'get-value':
+                revdb.send_answer(100, revdb.current_time(),
+                                       revdb.total_time())
+            if extra == 'go-fw':
+                revdb.go_forward(1, went_fw)
             ## if cmdline == 'set-break-after-0':
             ##     dbstate.break_after = 0
             ## if cmdline == 'print-id':
@@ -343,7 +331,7 @@
         lambda_blip = lambda: blip
         #
         class DBState:
-            break_after = -1
+            pass
         dbstate = DBState()
         #
         def main(argv):
@@ -352,9 +340,6 @@
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
                 revdb.stop_point()
-                ## if i == dbstate.break_after:
-                ##     revdb.send_output('breakpoint!\n')
-                ##     revdb.go_forward(1, _nothing, "")
                 print op
             return 9
         compile(cls, main, [], backendopt=False)
@@ -392,57 +377,17 @@
 
     def test_get_value(self):
         child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('__go 2')
-        child.expectx('(2)$ ')
-        child.sendline('r get-value')
-        child.expectx('<<>>\r\n'
-                      '2,1,3\r\n'
-                      'blipped\r\n'
-                      '(2)$ ')
+        child.send(Message(1, extra='get-value'))
+        child.expect(100, 1, 3)
 
     def test_go_fw(self):
         child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r go-fw')
-        child.expectx('<<>>\r\n'
-                      'blipped\r\n'
-                      'went-fw xx -> 2\r\n'
-                      'went-fw yy -> 3\r\n'
-                      '(3)$ ')
-
-    def test_change_time(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r change-time')
-        child.expectx('<<>>\r\n'
-                      'changed-time xyzzy -> 2\r\n'
-                      'went-fw zz -> 3\r\n'
-                      '(3)$ ')
-
-    def test_change_time_non_exact(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r change-time-non-exact')
-        child.expectx('<<>>\r\n'
-                      'changed-time nonx -> 1\r\n'
-                      'went-fw zz -> 2\r\n'
-                      'went-fw yy -> 3\r\n'
-                      '(3)$ ')
-
-    def test_dynamic_breakpoint(self):
-        py.test.skip("unsure if that's needed")
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r set-break-after-0')
-        child.expectx('(1)$ ')
-        child.sendline('__forward 5')
-        child.expectx('breakpoint!\r\n'
-                      '(2)$ ')
+        child.send(Message(1, extra='go-fw'))
+        child.expect(42, 1, -43, -44, 'go-fw')
+        child.expect(120, 2)
+        child.expect(120, 3)
+        child.send(Message(CMD_FORWARD, 0))
+        child.expect(ANSWER_STD, 3, Ellipsis)
 
     def test_get_unique_id_and_track_object(self):
         child = self.replay()

From pypy.commits at gmail.com  Sat Jun 18 03:04:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Jun 2016 00:04:04 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Start the Python-side
 command-line reverse debugger
Message-ID: <5764f264.49c51c0a.a3f77.ffffaaa7@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85224:31c1d222566e
Date: 2016-06-18 09:05 +0200
http://bitbucket.org/pypy/pypy/changeset/31c1d222566e/

Log:	Start the Python-side command-line reverse debugger

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -399,8 +399,8 @@
                     line += '\nPYPY_INHIBIT_TAIL_CALL();'
                     break
         elif self.db.reverse_debugger:
-            from rpython.translator.revdb import revdb_genc
-            line = revdb_genc.emit(line, self.lltypename(v_result), r)
+            from rpython.translator.revdb import gencsupp
+            line = gencsupp.emit(line, self.lltypename(v_result), r)
         return line
 
     def OP_DIRECT_CALL(self, op):
@@ -441,9 +441,9 @@
             if (S._gckind != 'gc' and not S._hints.get('is_excdata')
                     and not S._hints.get('static_immutable')
                     and not S._hints.get('ignore_revdb')):
-                from rpython.translator.revdb import revdb_genc
-                result = revdb_genc.emit(result, self.lltypename(op.result),
-                                         newvalue)
+                from rpython.translator.revdb import gencsupp
+                result = gencsupp.emit(result, self.lltypename(op.result),
+                                       newvalue)
         return result
 
     def generic_set(self, op, targetexpr):
@@ -455,8 +455,8 @@
         if self.db.reverse_debugger:
             S = self.lltypemap(op.args[0]).TO
             if S._gckind != 'gc' and not S._hints.get('is_excdata'):
-                from rpython.translator.revdb import revdb_genc
-                result = revdb_genc.emit_void(result)
+                from rpython.translator.revdb import gencsupp
+                result = gencsupp.emit_void(result)
         return result
 
     def OP_GETFIELD(self, op, ampersand=''):
@@ -586,8 +586,8 @@
             expr_result,
             is_atomic)
         if self.db.reverse_debugger:
-            from rpython.translator.revdb import revdb_genc
-            res += revdb_genc.record_malloc_uid(expr_result)
+            from rpython.translator.revdb import gencsupp
+            res += gencsupp.record_malloc_uid(expr_result)
         return res
 
     def OP_BOEHM_MALLOC(self, op):
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -160,8 +160,8 @@
             self.c_entrypoint_name = pfname
 
         if self.config.translation.reverse_debugger:
-            from rpython.translator.revdb import revdb_genc
-            revdb_genc.prepare_database(db)
+            from rpython.translator.revdb import gencsupp
+            gencsupp.prepare_database(db)
 
         for obj in exports.EXPORTS_obj2name.keys():
             db.getcontainernode(obj)
@@ -855,8 +855,8 @@
     if _CYGWIN:
         files.append(srcdir / 'cygwin_wait.c')
     if database.reverse_debugger:
-        from rpython.translator.revdb import revdb_genc
-        files += revdb_genc.extra_files()
+        from rpython.translator.revdb import gencsupp
+        files += gencsupp.extra_files()
     return eci.merge(ExternalCompilationInfo(separate_module_files=files))
 
 
diff --git a/rpython/translator/revdb/ancillary.py b/rpython/translator/revdb/ancillary.py
--- a/rpython/translator/revdb/ancillary.py
+++ b/rpython/translator/revdb/ancillary.py
@@ -1,5 +1,5 @@
+import py
 import os, sys
-from rpython.tool.udir import udir
 
 
 def build(tmpdir):
@@ -22,8 +22,12 @@
 
     ffibuilder.compile(tmpdir=tmpdir, verbose=True)
 
-def import_():
-    tmpdir = str(udir.ensure('ancillary', dir=1))
+def import_(verbose=False):
+    import rpython
+    basedir = py.path.local(rpython.__file__).dirpath()
+    tmpdir = str(basedir.ensure('_cache', 'ancillary', dir=1))
+    if verbose:
+        print tmpdir
     old_sys_path = sys.path[:]
     sys.path.insert(0, tmpdir)
     try:
@@ -50,4 +54,4 @@
 
 
 if __name__ == '__main__':
-    import_()
+    import_(verbose=True)
diff --git a/rpython/translator/revdb/revdb_genc.py b/rpython/translator/revdb/gencsupp.py
rename from rpython/translator/revdb/revdb_genc.py
rename to rpython/translator/revdb/gencsupp.py
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/interact.py
@@ -0,0 +1,76 @@
+import sys, re
+import subprocess, socket
+import traceback
+from rpython.translator.revdb.message import *
+
+r_cmdline = re.compile(r"(\S+)\s*(.*)")
+
+
+class RevDebugControl(object):
+
+    def __init__(self, revdb_log_filename, executable=None):
+        with open(revdb_log_filename, 'rb') as f:
+            header = f.readline()
+        fields = header.split('\t')
+        if len(fields) < 2 or fields[0] != 'RevDB:':
+            raise ValueError("file %r is not a RevDB log" % (
+                revdb_log_filename,))
+        if executable is None:
+            executable = fields[1]
+        #
+        s1, s2 = socket.socketpair()
+        subproc = subprocess.Popen(
+            [executable, '--revdb-replay', revdb_log_filename,
+             str(s2.fileno())])
+        s2.close()
+        self.subproc = subproc
+        child = ReplayProcess(subproc.pid, s1)
+        msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
+        self.total_stop_points = msg.arg2
+        child.expect(ANSWER_STD, 1, Ellipsis)
+        self.active_child = child
+        self.paused_children = {}
+
+    def interact(self):
+        last_command = ''
+        while True:
+            prompt = '(%d)$ ' % self.active_child.current_time()
+            try:
+                cmdline = raw_input(prompt).strip()
+            except EOFError:
+                print
+                cmdline = 'quit'
+            if not cmdline:
+                cmdline = last_command
+            match = r_cmdline.match(cmdline)
+            if not match:
+                continue
+            command, argument = match.groups()
+            try:
+                runner = getattr(self, 'command_' + command)
+            except AttributeError:
+                print >> sys.stderr, "no command '%s', try 'help'" % (command,)
+            else:
+                try:
+                    runner(argument)
+                except Exception as e:
+                    for line in traceback.format_exception_only(type(e), e):
+                        sys.stderr.write(line)
+                last_command = cmdline
+
+    def command_help(self, argument):
+        """Display commands summary"""
+        print 'Available commands:'
+        for name in dir(self):
+            if name.startswith('command_'):
+                command = name[len('command_'):]
+                docstring = getattr(self, name).__doc__ or 'undocumented'
+                print '\t%s\t%s' % (command, docstring)
+
+    def command_quit(self, argument):
+        """Exit the reverse debugger"""
+        sys.exit(0)
+
+    def command_go(self, argument):
+        """Go to time ARG"""
+        target_time = int(argument)
diff --git a/rpython/translator/revdb/revmsg.py b/rpython/translator/revdb/message.py
rename from rpython/translator/revdb/revmsg.py
rename to rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/revmsg.py
+++ b/rpython/translator/revdb/message.py
@@ -88,3 +88,13 @@
 
     def close(self):
         self.send(Message(CMD_QUIT))
+
+    def forward(self, steps):
+        self.send(Message(CMD_FORWARD, steps))
+        return self.expect(ANSWER_STD, Ellipsis, Ellipsis)
+
+    def current_time(self):
+        return self.forward(0).arg1
+
+    def currently_created_objects(self):
+        return self.forward(0).arg2
diff --git a/rpython/translator/revdb/revdb.py b/rpython/translator/revdb/revdb.py
new file mode 100755
--- /dev/null
+++ b/rpython/translator/revdb/revdb.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import sys, os
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description='Reverse debugger')
+    parser.add_argument('log', metavar='LOG', help='log file name')
+    parser.add_argument('-x', '--executable', dest='executable',
+                        help='name of the executable file '
+                             'that recorded the log')
+    options = parser.parse_args(sys.argv[1:])
+
+    sys.path.insert(0, os.path.abspath(
+        os.path.join(__file__, '..', '..', '..', '..')))
+
+    from rpython.translator.revdb.interact import RevDebugControl
+    ctrl = RevDebugControl(options.log, executable=options.executable)
+    ctrl.interact()
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -120,7 +120,7 @@
 
         write_all(RDB_SIGNATURE, strlen(RDB_SIGNATURE));
         for (i = 0; i < argc; i++) {
-            write_all(" ", 1);
+            write_all("\t", 1);
             write_all(argv[i], strlen(argv[i]));
         }
         write_all("\n\0", 2);
@@ -336,7 +336,7 @@
     }
     fprintf(stderr, "%s", RDB_SIGNATURE);
     while ((read_all(input, 1), input[0] != 0))
-        fputc(input[0], stderr);
+        fputc(input[0] == '\t' ? ' ' : input[0], stderr);
 
     read_all(&h, sizeof(h));
     if (h.version != RDB_VERSION) {
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -9,7 +9,7 @@
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.rtyper.lltypesystem import lltype, llmemory
 
-from rpython.translator.revdb.revmsg import *
+from rpython.translator.revdb.message import *
 
 
 class RDB(object):
@@ -17,7 +17,7 @@
         with open(filename, 'rb') as f:
             header = f.readline()
             self.buffer = f.read()
-        assert header == 'RevDB: ' + ' '.join(expected_argv) + '\n'
+        assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n'
         #
         self.cur = 0
         x = self.next('c'); assert x == '\x00'

From pypy.commits at gmail.com  Sat Jun 18 12:41:20 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sat, 18 Jun 2016 09:41:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Call _visit_starunpack in set
Message-ID: <576579b0.8438c20a.7991b.49a5@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85225:6e3952e1c5d7
Date: 2016-06-18 18:40 +0200
http://bitbucket.org/pypy/pypy/changeset/6e3952e1c5d7/

Log:	Call _visit_starunpack in set

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1017,7 +1017,7 @@
         ifexp.orelse.walkabout(self)
         self.use_next_block(end)
 
-    def _visit_list_or_tuple_starunpack(self, node, elts, ctx, single_op, inner_op, outer_op):
+    def _visit_starunpack(self, node, elts, ctx, single_op, inner_op, outer_op):
         elt_count = len(elts) if elts else 0
         seen_star = 0
         elt_subitems = 0
@@ -1042,8 +1042,7 @@
         else:
             self.emit_op_arg(single_op, seen_star)
     
-    #_visit_starunpack
-    def _visit_list_or_tuple_assignment(self, node, elts, ctx):
+    def _visit_assignment(self, node, elts, ctx):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
             seen_star = False
@@ -1073,18 +1072,18 @@
     def visit_Tuple(self, tup):
         self.update_position(tup.lineno)
         if tup.ctx == ast.Store:
-            self._visit_list_or_tuple_assignment(tup, tup.elts, tup.ctx)
+            self._visit_assignment(tup, tup.elts, tup.ctx)
         elif tup.ctx == ast.Load:
-            self._visit_list_or_tuple_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
+            self._visit_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
         else:
             self.visit_sequence(tup.elts)
 
     def visit_List(self, l):
         self.update_position(l.lineno)
         if l.ctx == ast.Store:
-            self._visit_list_or_tuple_assignment(l, l.elts, l.ctx)
+            self._visit_assignment(l, l.elts, l.ctx)
         elif l.ctx == ast.Load:
-            self._visit_list_or_tuple_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
+            self._visit_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
         else:
             self.visit_sequence(l.elts)
 
@@ -1111,12 +1110,14 @@
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
 
+    #def visit_Set(self, s):
+    #    self.update_position(s.lineno)
+    #    elt_count = len(s.elts) if s.elts is not None else 0
+    #    self.visit_sequence(s.elts)
+    #    self.emit_op_arg(ops.BUILD_SET, elt_count)
+    
     def visit_Set(self, s):
-        self.update_position(s.lineno)
-        elt_count = len(s.elts) if s.elts is not None else 0
-        self.visit_sequence(s.elts)
-        self.emit_op_arg(ops.BUILD_SET, elt_count)
-        #ops.BUILD_SET_UNPACK
+        self._visit_starunpack(s, s.elts, s.ctx, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
 
     def visit_Name(self, name):
         self.update_position(name.lineno)

From pypy.commits at gmail.com  Sat Jun 18 16:25:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 18 Jun 2016 13:25:00 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: Make sure that the GIL is reset
 properly when an unexpected exception is raised
Message-ID: <5765ae1c.c68e1c0a.c7b3b.ffffaa0e@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85226:0fdc10ae15b1
Date: 2016-06-18 21:24 +0100
http://bitbucket.org/pypy/pypy/changeset/0fdc10ae15b1/

Log:	Make sure that the GIL is reset properly when an unexpected
	exception is raised

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -800,6 +800,21 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
+def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+    from rpython.rlib import rgil
+    # see "Handling of the GIL" above
+    assert cpyext_glob_tid_ptr[0] == 0
+    if pygilstate_release:
+        from pypy.module.cpyext import pystate
+        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
+    else:
+        unlock = gil_release or _gil_auto
+    if unlock:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
+
+
 def make_wrapper_second_level(space, argtypesw, restype,
                               result_kind, error_value, gil):
     from rpython.rlib import rgil
@@ -827,6 +842,7 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
+        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -839,7 +855,6 @@
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
-            from pypy.module.cpyext import pystate
             if cpyext_glob_tid_ptr[0] == tid:
                 cpyext_glob_tid_ptr[0] = 0
                 args += (pystate.PyGILState_LOCKED,)
@@ -850,6 +865,10 @@
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
+        if pygilstate_release:
+            gilstate = rffi.cast(lltype.Signed, args[-1])
+        else:
+            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -919,24 +938,13 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
+            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        # see "Handling of the GIL" above
-        assert cpyext_glob_tid_ptr[0] == 0
-        if pygilstate_release:
-            from pypy.module.cpyext import pystate
-            arg = rffi.cast(lltype.Signed, args[-1])
-            unlock = (arg == pystate.PyGILState_UNLOCKED)
-        else:
-            unlock = gil_release or _gil_auto
-        if unlock:
-            rgil.release()
-        else:
-            cpyext_glob_tid_ptr[0] = tid
-
+        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -207,6 +207,7 @@
 PyGILState_STATE = rffi.INT
 PyGILState_LOCKED = 0
 PyGILState_UNLOCKED = 1
+PyGILState_IGNORE = 2
 
 ExecutionContext.cpyext_gilstate_counter_noleave = 0
 

From pypy.commits at gmail.com  Sat Jun 18 16:26:03 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sat, 18 Jun 2016 13:26:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix dict codegen
Message-ID: <5765ae5b.46c21c0a.6fb05.ffffadd9@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85227:58c84b553a11
Date: 2016-06-18 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/58c84b553a11/

Log:	Fix dict codegen

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1091,30 +1091,34 @@
         self.update_position(d.lineno)
         containers = 0
         elements = 0
+        is_unpacking = False
         if d.values:
             for i in range(len(d.values)):
-                if elements == 0xFFFF:
+                key = d.keys[i]
+                if key is None:
+                    is_unpacking = True
+                if elements == 0xFFFF or (elements and is_unpacking):
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
                     elements = 0
-                d.values[i].walkabout(self)
-                d.keys[i].walkabout(self)
-                elements += 1
+                if is_unpacking:
+                    d.values[i].walkabout(self)
+                    containers += 1
+                else:
+                    d.values[i].walkabout(self)
+                    d.keys[i].walkabout(self)
+                    elements += 1
         if elements or containers == 0:
             self.emit_op_arg(ops.BUILD_MAP, elements)
             containers += 1
         # If there is more than one dict, they need to be merged into
-        # a new dict.
-        while containers > 1:
+        # a new dict. If there is one dict and it's an unpacking, then
+        #it needs to be copied into a new dict.
+        while containers > 1 or is_unpacking:
             oparg = max(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
-
-    #def visit_Set(self, s):
-    #    self.update_position(s.lineno)
-    #    elt_count = len(s.elts) if s.elts is not None else 0
-    #    self.visit_sequence(s.elts)
-    #    self.emit_op_arg(ops.BUILD_SET, elt_count)
+            is_unpacking = 0
     
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, s.ctx, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)

From pypy.commits at gmail.com  Sun Jun 19 14:07:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Jun 2016 11:07:44 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: Fix an easy and a subtle GC
 issue
Message-ID: <5766df70.c61f1c0a.dd089.6a17@mx.google.com>

Author: Armin Rigo 
Branch: guard-compatible
Changeset: r85228:524da3e89aea
Date: 2016-06-19 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/524da3e89aea/

Log:	Fix an easy and a subtle GC issue

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -49,7 +49,8 @@
     def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth):
         # note that regalloc has a very similar compute, but
         # one that does iteration over all bindings, so slightly different,
-        # eh
+        # eh.  Also: for GUARD_COMPATIBLE, the gcmap we receive is not
+        # empty.  Make sure we only add some bits, but don't remove any.
         input_i = 0
         for i in range(len(failargs)):
             arg = failargs[i]
diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -25,6 +25,7 @@
 def _getofs(name):
     return llmemory.offsetof(BACKEND_CHOICES, name)
 BCFAILDESCR = _getofs('bc_faildescr')
+BCGCTABLETRACER = _getofs('bc_gc_table_tracer')
 BCMOSTRECENT = _getofs('bc_most_recent')
 BCLIST = _getofs('bc_list')
 del _getofs
@@ -46,6 +47,7 @@
 
 def bchoices_trace(gc, obj_addr, callback, arg):
     gc._trace_callback(callback, arg, obj_addr + BCFAILDESCR)
+    gc._trace_callback(callback, arg, obj_addr + BCGCTABLETRACER)
     bchoices_pair(gc, obj_addr + BCMOSTRECENT, callback, arg)
     length = (obj_addr + BCLIST + BCLISTLENGTHOFS).signed[0]
     array_addr = obj_addr + BCLIST + BCLISTITEMSOFS
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1937,7 +1937,15 @@
 
     def implement_guard_recovery(self, guard_opnum, faildescr, failargs,
                                  fail_locs, frame_depth):
-        gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
+        if guard_opnum == rop.GUARD_COMPATIBLE:
+            # In this case, we might stop at the guard but continue
+            # running anyway.  So we must make sure that the gcmap
+            # ensures that all gcrefs stay alive.
+            gcmap = self._regalloc.get_gcmap()
+        else:
+            # Common case: get an empty gcmap.  It will be filled
+            # with compute_gcmap() in llsupport.assembler.
+            gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
         faildescrindex = self.get_gcref_from_faildescr(faildescr)
         return GuardToken(self.cpu, gcmap, faildescr, failargs, fail_locs,
                           guard_opnum, frame_depth, faildescrindex)

From pypy.commits at gmail.com  Sun Jun 19 14:07:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Jun 2016 11:07:46 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: Blindly port the fix to the
 other backends
Message-ID: <5766df72.cc9d1c0a.d330b.fffffa7c@mx.google.com>

Author: Armin Rigo 
Branch: guard-compatible
Changeset: r85229:9840b5902d7e
Date: 2016-06-19 19:14 +0100
http://bitbucket.org/pypy/pypy/changeset/9840b5902d7e/

Log:	Blindly port the fix to the other backends

diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -179,7 +179,15 @@
         descr = op.getdescr()
         assert isinstance(descr, AbstractFailDescr)
 
-        gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
+        if op.getopnum() == rop.GUARD_COMPATIBLE:
+            # In this case, we might stop at the guard but continue
+            # running anyway.  So we must make sure that the gcmap
+            # ensures that all gcrefs stay alive.
+            gcmap = self._regalloc.get_gcmap()
+        else:
+            # Common case: get an empty gcmap.  It will be filled
+            # with compute_gcmap() in llsupport.assembler.
+            gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
         faildescrindex = self.get_gcref_from_faildescr(descr)
         token = ArmGuardToken(self.cpu, gcmap,
                                     descr,
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -275,7 +275,15 @@
 
     def build_guard_token(self, op, frame_depth, arglocs, fcond):
         descr = op.getdescr()
-        gcmap = allocate_gcmap(self, frame_depth, r.JITFRAME_FIXED_SIZE)
+        if op.getopnum() == rop.GUARD_COMPATIBLE:
+            # In this case, we might stop at the guard but continue
+            # running anyway.  So we must make sure that the gcmap
+            # ensures that all gcrefs stay alive.
+            gcmap = self._regalloc.get_gcmap()
+        else:
+            # Common case: get an empty gcmap.  It will be filled
+            # with compute_gcmap() in llsupport.assembler.
+            gcmap = allocate_gcmap(self, frame_depth, r.JITFRAME_FIXED_SIZE)
         faildescrindex = self.get_gcref_from_faildescr(descr)
         token = PPCGuardToken(self.cpu, gcmap, descr, op.getfailargs(),
                               arglocs, op.getopnum(), frame_depth,
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -636,7 +636,15 @@
 
     def build_guard_token(self, op, frame_depth, arglocs, fcond):
         descr = op.getdescr()
-        gcmap = allocate_gcmap(self, frame_depth, r.JITFRAME_FIXED_SIZE)
+        if op.getopnum() == rop.GUARD_COMPATIBLE:
+            # In this case, we might stop at the guard but continue
+            # running anyway.  So we must make sure that the gcmap
+            # ensures that all gcrefs stay alive.
+            gcmap = self._regalloc.get_gcmap()
+        else:
+            # Common case: get an empty gcmap.  It will be filled
+            # with compute_gcmap() in llsupport.assembler.
+            gcmap = allocate_gcmap(self, frame_depth, r.JITFRAME_FIXED_SIZE)
         faildescrindex = self.get_gcref_from_faildescr(descr)
         token = ZARCHGuardToken(self.cpu, gcmap, descr, op.getfailargs(),
                               arglocs, op.getopnum(), frame_depth,

From pypy.commits at gmail.com  Sun Jun 19 15:59:59 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Sun, 19 Jun 2016 12:59:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: (insert note for bug with depth is 1)
Message-ID: <5766f9bf.692dc20a.489bc.ffff9abe@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85230:74e75cb5c5eb
Date: 2016-06-19 21:59 +0200
http://bitbucket.org/pypy/pypy/changeset/74e75cb5c5eb/

Log:	(insert note for bug with depth is 1)

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -397,6 +397,7 @@
         for block in blocks:
             depth = self._do_stack_depth_walk(block)
             if block.auto_inserted_return and depth != 0:
+                #depth is 1, import pdb;pdb.pm()
                 os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
                     self.compile_info.filename, self.name, self.first_lineno))
                 raise StackDepthComputationError   # fatal error
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1353,8 +1353,9 @@
         w_sum = self.space.newset()
         num_maps = itemcount & 0xff
         function_location = (itemcount >> 8) & 0xff
-        for i in range(itemcount, 0, -1):
+        for i in range(num_maps, 0, -1):
             arg = self.space.peek(i)
+            # intersection = _dictviews_and(w_sum, arg) #check after bugs are done
             self.space.call_method(w_sum, 'update', self.space.peek(i))
         while itemcount != 0:
             self.popvalue()

From pypy.commits at gmail.com  Mon Jun 20 03:06:08 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Jun 2016 00:06:08 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: first vector loop successfully
 compiles on ppc (floating point only)
Message-ID: <576795e0.692dc20a.489bc.445b@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85231:1352b56d157d
Date: 2016-06-20 09:05 +0200
http://bitbucket.org/pypy/pypy/changeset/1352b56d157d/

Log:	first vector loop successfully compiles on ppc (floating point only)

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -61,7 +61,8 @@
 XFL = Form("FM", "frB", "XO1", "Rc")
 XFX = Form("CRM", "rS", "XO1")
 XLL = Form("LL", "XO1")
-XX1 = Form("vrT", "rA", "rB", "XO1")
+XX1 = Form("fvrT", "rA", "rB", "XO1")
+XX3 = Form("fvrT", "fvrA", "fvrB", "XO9")
 VX = Form("lvrT", "lvrA", "lvrB", "XO8")
 
 MI = Form("rA", "rS", "SH", "MB", "ME", "Rc")
@@ -576,6 +577,12 @@
 class PPCVSXAssembler(object):
     _mixin_ = True
 
+    # floating point operations (ppc got it's own vector
+    # unit for double/single precision floating points
+
+    # FLOAT
+    # -----
+
     # load
     lxvdsx = XX1(31, XO1=332) # splat first element
     lxvd2x = XX1(31, XO1=844)
@@ -585,7 +592,23 @@
     stxvd2x = XX1(31, XO1=972)
     stxvw4x = XX1(31, XO1=908)
 
-    # integer
+    # arith
+
+    # add
+    xvadddp = XX3(60, XO9=96)
+    xvaddsp = XX3(60, XO9=64)
+    # sub
+    xvsubdp = XX3(60, XO9=104)
+    xvsubsp = XX3(60, XO9=72)
+    # mul
+    xvmuldp = XX3(60, XO9=112)
+    xvmulsp = XX3(60, XO9=80)
+    # div
+    xvdivdp = XX3(60, XO9=102)
+    xvdivsp = XX3(60, XO9=88)
+
+    # INTEGER
+    # -------
     vaddudm = VX(4, XO8=192)
 
 class PPCAssembler(BasicPPCAssembler, PPCVSXAssembler):
diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py
--- a/rpython/jit/backend/ppc/ppc_field.py
+++ b/rpython/jit/backend/ppc/ppc_field.py
@@ -43,7 +43,9 @@
     "spr":    (11, 20),
     "TO":     ( 6, 10),
     "UIMM":   (16, 31),
-    "vrT":    (6,  31, 'unsigned', regname._V, 'overlap'),
+    "fvrT":   (6,  31, 'unsigned', regname._V, 'overlap'),
+    "fvrA":   (11, 31, 'unsigned', regname._V, 'overlap'),
+    "fvrB":   (16, 31, 'unsigned', regname._V, 'overlap'),
     # low vector register T (low in a sense:
     # can only address 32 vector registers)
     "lvrT":   (6,  10, 'unsigned', regname._V),
@@ -59,6 +61,7 @@
     "XO6":    (21, 29),
     "XO7":    (27, 30),
     "XO8":    (21, 31),
+    "XO9":    (21, 28),
     "LL":     ( 9, 10),
 }
 
@@ -110,18 +113,6 @@
         value = super(sh, self).decode(inst)
         return (value & 32) << 5 | (value >> 10 & 31)
 
-# ??? class tx(Field):
-# ???     def encode(self, value):
-# ???         value = (value & 31) << 20 | (value & 32) >> 5
-# ???         return super(tx, self).encode(value)
-# ???     def decode(self, inst):
-# ???         value = super(tx, self).decode(inst)
-# ???         return (value & 32) << 5 | (value >> 20 & 31)
-# ???     def r(self):
-# ???         import pdb; pdb.set_trace()
-# ???         return super(tx, self).r()
-# other special fields?
-
 ppc_fields = {
     "LI":  IField("LI", *fields["LI"]),
     "BD":  IField("BD", *fields["BD"]),
@@ -129,7 +120,6 @@
     "mbe": mbe("mbe",   *fields["mbe"]),
     "sh":  sh("sh",     *fields["sh"]),
     "spr": spr("spr",   *fields["spr"]),
-    # ??? "vrT": tx("vrT",    *fields["vrT"]),
 }
 
 for f in fields:
diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py
--- a/rpython/jit/backend/ppc/rassemblermaker.py
+++ b/rpython/jit/backend/ppc/rassemblermaker.py
@@ -46,9 +46,15 @@
         elif field.name == 'sh':
             body.append('sh1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value))
             value = 'sh1'
-        elif field.name == 'vrT':
+        elif field.name == 'fvrT':
             body.append('vrT1 = (%s & 31) << 21 | (%s & 32) >> 5' % (value, value))
             value = 'vrT1'
+        elif field.name == 'fvrA':
+            body.append('fvrA1 = ((%s & 31) << 15 | (%s & 32) >> 5) << 2' % (value, value))
+            value = 'fvrA1'
+        elif field.name == 'fvrB':
+            body.append('fvrB1 = ((%s & 31) << 10 | (%s & 32) >> 5) << 1' % (value, value))
+            value = 'fvrB1'
         if isinstance(field, IField):
             body.append('v |= ((%3s >> 2) & r_uint(%#05x)) << 2' % (value, field.mask))
         else:
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -48,19 +48,12 @@
 
     def _vec_load(self, resloc, baseloc, indexloc, integer, itemsize, aligned):
         if integer:
+            raise NotImplementedError
+        else:
             if itemsize == 4:
                 self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
             elif itemsize == 8:
                 self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
-            else:
-                raise NotImplementedError
-        else:
-            if itemsize == 4:
-                self.mc.MOVUPS(resloc, src_addr)
-            elif itemsize == 8:
-                self.mc.MOVUPD(resloc, src_addr)
-            else:
-                raise NotImplementedError
 
     def _emit_vec_setitem(self, op, arglocs, regalloc):
         # prepares item scale (raw_store does not)
@@ -83,14 +76,12 @@
 
     def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, aligned):
         if integer:
+            raise NotImplementedError
+        else:
             if itemsize == 4:
                 self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
             elif itemsize == 8:
                 self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
-            else:
-                raise NotImplementedError
-        else:
-            raise NotImplementedError
 
 
     def emit_vec_int_add(self, op, arglocs, regalloc):
@@ -103,8 +94,41 @@
         elif size == 4:
             raise NotImplementedError
         elif size == 8:
+            raise NotImplementedError # need value in another register!
             self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
 
+    def emit_vec_float_add(self, op, arglocs, resloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvaddsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvadddp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_sub(self, op, arglocs, resloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvsubsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvsubdp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_mul(self, op, arglocs, resloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvmulsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvmuldp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_truediv(self, op, arglocs, resloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvdivsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
+
     #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
     #    self.implement_guard(guard_token)
 
@@ -253,23 +277,6 @@
     #def genop_vec_int_xor(self, op, arglocs, resloc):
     #    self.mc.PXOR(resloc, arglocs[0])
 
-    #genop_vec_float_arith = """
-    #def genop_vec_float_{type}(self, op, arglocs, resloc):
-    #    loc0, loc1, itemsize_loc = arglocs
-    #    itemsize = itemsize_loc.value
-    #    if itemsize == 4:
-    #        self.mc.{p_op_s}(loc0, loc1)
-    #    elif itemsize == 8:
-    #        self.mc.{p_op_d}(loc0, loc1)
-    #"""
-    #for op in ['add','mul','sub']:
-    #    OP = op.upper()
-    #    _source = genop_vec_float_arith.format(type=op,
-    #                                           p_op_s=OP+'PS',
-    #                                           p_op_d=OP+'PD')
-    #    exec py.code.Source(_source).compile()
-    #del genop_vec_float_arith
-
     #def genop_vec_float_truediv(self, op, arglocs, resloc):
     #    loc0, loc1, sizeloc = arglocs
     #    size = sizeloc.value
@@ -569,10 +576,10 @@
     prepare_vec_int_add = prepare_vec_arith
     #prepare_vec_int_sub = prepare_vec_arith
     #prepare_vec_int_mul = prepare_vec_arith
-    #prepare_vec_float_add = prepare_vec_arith
-    #prepare_vec_float_sub = prepare_vec_arith
-    #prepare_vec_float_mul = prepare_vec_arith
-    #prepare_vec_float_truediv = prepare_vec_arith
+    prepare_vec_float_add = prepare_vec_arith
+    prepare_vec_float_sub = prepare_vec_arith
+    prepare_vec_float_mul = prepare_vec_arith
+    prepare_vec_float_truediv = prepare_vec_arith
     del prepare_vec_arith
 
     def _prepare_vec_store(self, op):
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -1,5 +1,6 @@
 import py
-
+import pytest
+import math
 from hypothesis import given, note, strategies as st
 from rpython.jit.metainterp.warmspot import ll_meta_interp, get_stats
 from rpython.jit.metainterp.test.support import LLJitMixin
@@ -13,7 +14,8 @@
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem,
                                      free_raw_storage, raw_storage_getitem)
-from rpython.rlib.objectmodel import specialize, is_annotation_constant
+from rpython.rlib.objectmodel import (specialize, is_annotation_constant,
+        always_inline)
 from rpython.jit.backend.detect_cpu import getcpuclass
 
 CPU = getcpuclass()
@@ -24,7 +26,40 @@
 def free(mem):
     lltype.free(mem, flavor='raw')
 
+def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
+    return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
+
+class RawStorage(object):
+    def __init__(self):
+        self.arrays = []
+
+    def new(self, values, type, size=None, zero=True):
+        bytecount = rffi.sizeof(type)
+        if not values:
+            array = alloc_raw_storage(size*bytecount, zero=zero)
+            self.arrays.append(array)
+            return array
+        else:
+            size = len(values)*bytecount
+            array = alloc_raw_storage(size, zero=zero)
+            for i,v in enumerate(values):
+                raw_storage_setitem(array, i*bytecount, rffi.cast(type,v))
+            self.arrays.append(array)
+            return array
+
+    def clear(self):
+        while self.arrays:
+            array = self.arrays.pop()
+            free_raw_storage(array)
+
+ at pytest.fixture(scope='session')
+def rawstorage(request):
+    rs = RawStorage()
+    request.addfinalizer(rs.clear)
+    return rs
+
 integers_64bit = st.integers(min_value=-2**63, max_value=2**63-1)
+floats = st.floats()
 
 class VectorizeTests:
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
@@ -40,42 +75,80 @@
                               type_system=self.type_system,
                               vec=vec, vec_all=vec_all)
 
-    @given(st.lists(integers_64bit, min_size=5, max_size=50),
-           st.lists(integers_64bit, min_size=5, max_size=50))
-    def test_vector_simple(self, la, lb):
-        myjitdriver = JitDriver(greens = [],
-                                reds = 'auto',
-                                vectorize=True)
-        i = min(len(la), len(lb))
-        la = la[:i]
-        lb = lb[:i]
-        bc = i*rffi.sizeof(rffi.SIGNED)
-        vc = alloc_raw_storage(bc, zero=True)
+    @given(data=st.data())
+    @pytest.mark.parametrize('func', [lambda a,b: a+b,
+        lambda a,b: a*b, lambda a,b: a-b, lambda a,b: a / b])
+    def test_vector_simple_float(self, func, data):
+        func = always_inline(func)
+
+        type = rffi.DOUBLE
+        size = rffi.sizeof(rffi.DOUBLE)
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            while i < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(type,va,i)
+                b = raw_storage_getitem(type,vb,i)
+                c = func(a,b)
+                raw_storage_setitem(vc, i, rffi.cast(type,c))
+                i += size
+
+        la = data.draw(st.lists(floats, min_size=10, max_size=150))
+        #la = [0.0,0.0,0.0,0.0,0.0,0.0,0.0]
+        #lb = [0.0,0.0,0.0,0.0,1.7976931348623157e+308,0.0,0.0]
+        l = len(la)
+        lb = data.draw(st.lists(floats, min_size=l, max_size=l))
+
+        rawstorage = RawStorage()
+        va = rawstorage.new(la, type)
+        vb = rawstorage.new(lb, type)
+        vc = rawstorage.new(None, type, size=l)
+        self.meta_interp(f, [l*size, va, vb, vc])
+
+        for i in range(l):
+            c = raw_storage_getitem(type,vc,i*size)
+            r = func(la[i], lb[i])
+            assert isclose(r, c) or (math.isnan(r) and math.isnan(c)) or \
+                   (math.isinf(r) and math.isinf(c) and \
+                    (r < 0.0 and c < 0.0) or \
+                    (r > 0.0 and c > 0.0))
+
+        rawstorage.clear()
+
+    #@given(st.data())
+    def test_vector_simple_int(self):
+
+        type = rffi.SIGNED
         size = rffi.sizeof(rffi.SIGNED)
-        def f(d):
-            va = alloc_raw_storage(bc, zero=True)
-            vb = alloc_raw_storage(bc, zero=True)
-            x = 1
-            for i in range(d):
-                j = i*size
-                raw_storage_setitem(va, j, rffi.cast(rffi.SIGNED,la[i]))
-                raw_storage_setitem(vb, j, rffi.cast(rffi.SIGNED,lb[i]))
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
             i = 0
-            while i < bc:
+            while i < bytecount:
                 myjitdriver.jit_merge_point()
-                a = raw_storage_getitem(rffi.SIGNED,va,i)
-                b = raw_storage_getitem(rffi.SIGNED,vb,i)
+                a = raw_storage_getitem(type,va,i)
+                b = raw_storage_getitem(type,vb,i)
                 c = a+b
-                raw_storage_setitem(vc, i, rffi.cast(rffi.SIGNED,c))
-                i += 1*size
+                raw_storage_setitem(vc, i, rffi.cast(type,c))
+                i += size
 
-            free_raw_storage(va)
-            free_raw_storage(vb)
-        self.meta_interp(f, [i])
-        for p in range(i):
-            c = raw_storage_getitem(rffi.SIGNED,vc,p*size)
-            assert intmask(la[p] + lb[p]) == c
-        free_raw_storage(vc)
+        rawstorage = RawStorage()
+        #la = data.draw(st.lists(integers_64bit, min_size=10, max_size=150))
+        la = [0] * 10
+        l = len(la)
+        #lb = data.draw(st.lists(integers_64bit, min_size=l, max_size=l))
+        lb = [0] * 10
+
+        va = rawstorage.new(la, lltype.Signed)
+        vb = rawstorage.new(lb, lltype.Signed)
+        vc = rawstorage.new(None, lltype.Signed, size=l)
+        self.meta_interp(f, [l*size, va, vb, vc])
+
+        for i in range(l):
+            c = raw_storage_getitem(type,vc,i*size)
+            assert intmask(la[i] + lb[i]) == c
+
+        rawstorage.clear()
 
     @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,131,142,143])
     def test_vectorize_array_get_set(self,i):

From pypy.commits at gmail.com  Mon Jun 20 03:26:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 00:26:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <57679aaf.09f6c20a.38379.34ee@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85232:7b34a47bb51b
Date: 2016-06-20 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/7b34a47bb51b/

Log:	in-progress

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -27,14 +27,26 @@
         child = ReplayProcess(subproc.pid, s1)
         msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
         self.total_stop_points = msg.arg2
-        child.expect(ANSWER_STD, 1, Ellipsis)
+        msg = child.expect(ANSWER_STD, 1, Ellipsis)
+        child.update_times(msg)
         self.active_child = child
-        self.paused_children = {}
+        self.paused_children = []
+        #
+        # fixed time division for now
+        if self.total_stop_points < 1:
+            raise ValueError("no stop points recorded in %r" % (
+                revdb_log_filename,))
+        self.fork_points = {1: child.clone()}
+        p = 1
+        while p < self.total_stop_points and len(self.fork_points) < 30:
+            p += int((self.total_stop_points - p) * 0.25) + 1
+            self.fork_points[p] = None
 
     def interact(self):
         last_command = ''
         while True:
-            prompt = '(%d)$ ' % self.active_child.current_time()
+            last_time = self.active_child.current_time
+            prompt = '(%d)$ ' % last_time
             try:
                 cmdline = raw_input(prompt).strip()
             except EOFError:
@@ -54,8 +66,7 @@
                 try:
                     runner(argument)
                 except Exception as e:
-                    for line in traceback.format_exception_only(type(e), e):
-                        sys.stderr.write(line)
+                    traceback.print_exc()
                 last_command = cmdline
 
     def command_help(self, argument):
@@ -68,9 +79,26 @@
                 print '\t%s\t%s' % (command, docstring)
 
     def command_quit(self, argument):
-        """Exit the reverse debugger"""
+        """Exit the debugger"""
         sys.exit(0)
 
+    def change_child(self, target_time):
+        """If 'target_time' is not soon after 'self.active_child',
+        close it and fork another child."""
+        assert 1 <= target_time <= self.total_stop_points
+        best = max([key for key in self.fork_points if key <= target_time])
+        if best < self.active_child.current_time <= target_time:
+            pass     # keep 'active_child'
+        else:
+            self.active_child.close()
+            self.active_child = self.fork_points[best].clone()
+
     def command_go(self, argument):
         """Go to time ARG"""
         target_time = int(argument)
+        if target_time < 1:
+            target_time = 1
+        if target_time > self.total_stop_points:
+            target_time = self.total_stop_points
+        self.change_child(target_time)
+        self.active_child.forward(target_time - self.active_child.current_time)
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -1,8 +1,5 @@
-import os, struct, socket, errno
-from rpython.translator.revdb import ancillary
 
-
-INIT_VERSION_NUMBER   = 0xd80100
+INIT_VERSION_NUMBER = 0xd80100
 
 CMD_FORK     = -1
 CMD_QUIT     = -2
@@ -15,6 +12,10 @@
 
 
 class Message(object):
+    """Represent messages sent and received to subprocesses
+    started with --revdb-replay.
+    """
+
     def __init__(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
         self.cmd = cmd
         self.arg1 = arg1
@@ -36,65 +37,3 @@
 
     def __ne__(self, other):
         return not (self == other)
-
-
-class ReplayProcess(object):
-    def __init__(self, pid, control_socket):
-        self.pid = pid
-        self.control_socket = control_socket
-
-    def _recv_all(self, size):
-        pieces = []
-        while size > 0:
-            data = self.control_socket.recv(size)
-            if not data:
-                raise EOFError
-            size -= len(data)
-            pieces.append(data)
-        return ''.join(pieces)
-
-    def send(self, msg):
-        binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
-                             msg.arg1, msg.arg2, msg.arg3)
-        self.control_socket.sendall(binary + msg.extra)
-
-    def recv(self):
-        binary = self._recv_all(struct.calcsize("iIqqq"))
-        cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
-        extra = self._recv_all(size)
-        return Message(cmd, arg1, arg2, arg3, extra)
-
-    def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
-        msg = self.recv()
-        assert msg.cmd == cmd
-        if arg1 is not Ellipsis:
-            assert msg.arg1 == arg1
-        if arg2 is not Ellipsis:
-            assert msg.arg2 == arg2
-        if arg3 is not Ellipsis:
-            assert msg.arg3 == arg3
-        if extra is not Ellipsis:
-            assert msg.extra == extra
-        return msg
-
-    def fork(self):
-        self.send(Message(CMD_FORK))
-        s1, s2 = socket.socketpair()
-        ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
-        s2.close()
-        msg = self.expect(ANSWER_FORKED, Ellipsis)
-        child_pid = msg.arg1
-        return ReplayProcess(child_pid, s1)
-
-    def close(self):
-        self.send(Message(CMD_QUIT))
-
-    def forward(self, steps):
-        self.send(Message(CMD_FORWARD, steps))
-        return self.expect(ANSWER_STD, Ellipsis, Ellipsis)
-
-    def current_time(self):
-        return self.forward(0).arg1
-
-    def currently_created_objects(self):
-        return self.forward(0).arg2
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/process.py
@@ -0,0 +1,168 @@
+import os, struct, socket, errno, subprocess
+from rpython.translator.revdb import ancillary
+from rpython.translator.revdb.message import *
+
+
+class ReplayProcess(object):
+    """Represent one replaying subprocess.
+
+    It can be either the one started with --revdb-replay, or a fork.
+    """
+
+    def __init__(self, pid, control_socket):
+        self.pid = pid
+        self.control_socket = control_socket
+
+    def _recv_all(self, size):
+        pieces = []
+        while size > 0:
+            data = self.control_socket.recv(size)
+            if not data:
+                raise EOFError
+            size -= len(data)
+            pieces.append(data)
+        return ''.join(pieces)
+
+    def send(self, msg):
+        binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
+                             msg.arg1, msg.arg2, msg.arg3)
+        self.control_socket.sendall(binary + msg.extra)
+
+    def recv(self):
+        binary = self._recv_all(struct.calcsize("iIqqq"))
+        cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
+        extra = self._recv_all(size)
+        return Message(cmd, arg1, arg2, arg3, extra)
+
+    def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
+        msg = self.recv()
+        assert msg.cmd == cmd
+        if arg1 is not Ellipsis:
+            assert msg.arg1 == arg1
+        if arg2 is not Ellipsis:
+            assert msg.arg2 == arg2
+        if arg3 is not Ellipsis:
+            assert msg.arg3 == arg3
+        if extra is not Ellipsis:
+            assert msg.extra == extra
+        return msg
+
+    def update_times(self, msg):
+        self.current_time = msg.arg1
+        self.currently_created_objects = msg.arg2
+
+    def clone(self):
+        """Fork this subprocess.  Returns a new ReplayProcess() that is
+        an identical copy.
+        """
+        self.send(Message(CMD_FORK))
+        s1, s2 = socket.socketpair()
+        ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
+        s2.close()
+        msg = self.expect(ANSWER_FORKED, Ellipsis, Ellipsis, Ellipsis)
+        self.update_times(msg)
+        child_pid = msg.arg3
+        other = ReplayProcess(child_pid, s1)
+        other.update_times(msg)
+        return other
+
+    def close(self):
+        """Close this subprocess."""
+        self.send(Message(CMD_QUIT))
+
+    def forward(self, steps):
+        """Move this subprocess forward in time."""
+        self.send(Message(CMD_FORWARD, steps))
+        msg = self.expect(ANSWER_STD, Ellipsis, Ellipsis)
+        self.update_times(msg)
+        return msg
+
+
+class ReplayProcessGroup(object):
+    """Handle a family of subprocesses.
+    """
+    MAX_SUBPROCESSES = 31       # maximum number of subprocesses
+    STEP_RATIO = 0.25           # subprocess n is between subprocess n-1
+                                #   and the end, at this time fraction
+
+    def __init__(self, executable, revdb_log_filename):
+        s1, s2 = socket.socketpair()
+        initial_subproc = subprocess.Popen(
+            [executable, '--revdb-replay', revdb_log_filename,
+             str(s2.fileno())])
+        s2.close()
+        child = ReplayProcess(initial_subproc.pid, s1)
+        msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
+        self.total_stop_points = msg.arg2
+        msg = child.expect(ANSWER_STD, 1, Ellipsis)
+        child.update_times(msg)
+
+        self.active = child
+        self.paused = {1: child.clone()}     # {time: subprocess}
+
+    def get_current_time(self):
+        return self.active.current_time
+
+    def _check_current_time(self, time):
+        assert self.get_current_time() == time
+        self.active.send(Message(CMD_FORWARD, 0))
+        return self.active.expect(ANSWER_STD, time, Ellipsis)
+
+    def get_max_time(self):
+        return self.total_stop_points
+
+    def get_next_clone_time(self):
+        if len(self.paused) >= self.MAX_SUBPROCESSES:
+            next_time = self.total_stop_points + 1
+        else:
+            latest_done = max(self.paused)
+            range_not_done = self.total_stop_points - latest_done
+            next_time = latest_done + int(self.STEP_RATIO * range_not_done) + 1
+        return next_time
+
+    def forward(self, steps):
+        """Go forward, for the given number of 'steps' of time.
+
+        If needed, it will leave clones at intermediate times.
+        Does not close the active subprocess.
+        """
+        assert steps >= 0
+        while True:
+            cur_time = self.get_current_time()
+            if cur_time + steps > self.total_stop_points:
+                steps = self.total_stop_points - cur_time
+            next_clone = self.get_next_clone_time()
+            rel_next_clone = next_clone - cur_time
+            if rel_next_clone > steps:
+                break
+            assert rel_next_clone >= 0
+            if rel_next_clone > 0:
+                self.active.forward(rel_next_clone)
+                steps -= rel_next_clone
+            clone = self.active.clone()
+            self.paused[clone.current_time] = clone
+        self.active.forward(steps)
+
+    def _resume(self, from_time):
+        clone_me = self.paused[from_time]
+        self.active.close()
+        self.active = clone_me.clone()
+
+    def jump_in_time(self, target_time):
+        """Jump in time at the given 'target_time'.
+
+        This function can close the active subprocess.
+        """
+        if target_time < 1:
+            target_time = 1
+        if target_time > self.total_stop_points:
+            target_time = self.total_stop_points
+
+        cur_time = self.get_current_time()
+        if target_time >= cur_time:    # can go forward
+            if cur_time >= max(self.paused):  # current time is past all forks
+                self.forward(target_time - cur_time)
+                return
+        # else, start from a fork
+        self._resume(max(time for time in self.paused if time <= target_time))
+        self.forward(target_time - self.get_current_time())
diff --git a/rpython/translator/revdb/revdb.py b/rpython/translator/revdb/revdb.py
--- a/rpython/translator/revdb/revdb.py
+++ b/rpython/translator/revdb/revdb.py
@@ -10,7 +10,7 @@
     parser.add_argument('-x', '--executable', dest='executable',
                         help='name of the executable file '
                              'that recorded the log')
-    options = parser.parse_args(sys.argv[1:])
+    options = parser.parse_args()
 
     sys.path.insert(0, os.path.abspath(
         os.path.join(__file__, '..', '..', '..', '..')))
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -282,8 +282,9 @@
 
 static void answer_std(void)
 {
-    write_answer(ANSWER_STD, rpy_revdb.stop_point_seen,
-                 rpy_revdb.unique_id_seen, 0);
+    assert(stopped_time != 0);
+    assert(stopped_uid != 0);
+    write_answer(ANSWER_STD, stopped_time, stopped_uid, 0);
 }
 
 static RPyString *make_rpy_string(size_t length)
@@ -516,7 +517,7 @@
     }
     else {
         /* in the parent */
-        write_answer(ANSWER_FORKED, child_pid, 0, 0);
+        write_answer(ANSWER_FORKED, stopped_time, stopped_uid, child_pid);
         close(child_sockfd);
     }
 }
@@ -593,6 +594,7 @@
                 break;
             }
         }
+        rpy_revdb.stop_point_seen = stopped_time;
         rpy_revdb.unique_id_seen = stopped_uid;
         stopped_time = 0;
         stopped_uid = 0;
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -10,6 +10,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 
 from rpython.translator.revdb.message import *
+from rpython.translator.revdb.process import ReplayProcess
 
 
 class RDB(object):
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/test/test_process.py
@@ -0,0 +1,49 @@
+from rpython.rlib import revdb
+from rpython.translator.revdb.message import *
+from rpython.translator.revdb.process import ReplayProcessGroup
+
+from hypothesis import given, strategies
+
+
+class TestReplayProcessGroup:
+
+    def setup_class(cls):
+        from rpython.translator.revdb.test.test_basic import compile, run
+
+        class Stuff:
+            pass
+
+        class DBState:
+            pass
+        dbstate = DBState()
+
+        def main(argv):
+            for i, op in enumerate(argv[1:]):
+                dbstate.stuff = Stuff()
+                dbstate.stuff.x = i + 1000
+                revdb.stop_point()
+                print op
+            return 9
+        compile(cls, main, [], backendopt=False)
+        assert run(cls, 'abc d ef g h i j k l m') == (
+            'abc\nd\nef\ng\nh\ni\nj\nk\nl\nm\n')
+
+
+    def test_init(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        assert group.get_max_time() == 10
+        assert group.get_next_clone_time() == 4
+
+    def test_forward(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        group.forward(100)
+        assert group.get_current_time() == 10
+        assert sorted(group.paused) == [1, 4, 6, 8, 9, 10]
+        assert group._check_current_time(10)
+
+    @given(strategies.lists(strategies.integers(min_value=1, max_value=10)))
+    def test_jump_in_time(self, target_times):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        for target_time in target_times:
+            group.jump_in_time(target_time)
+            group._check_current_time(target_time)

From pypy.commits at gmail.com  Mon Jun 20 04:05:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 01:05:45 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Breakpoint
Message-ID: <5767a3d9.cc9d1c0a.d330b.ffffdbe9@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85233:29dbfbbe1024
Date: 2016-06-20 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/29dbfbbe1024/

Log:	Breakpoint

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -55,6 +55,9 @@
     """
     _change_time('f', time_delta, callback)
 
+def breakpoint(num):
+    llop.revdb_breakpoint(lltype.Void, num)
+
 @specialize.argtype(0)
 def get_unique_id(x):
     """Returns the creation number of the object 'x'.  For objects created
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -568,6 +568,7 @@
     'revdb_stop_point':     LLOp(),
     'revdb_send_answer':    LLOp(),
     'revdb_change_time':    LLOp(),
+    'revdb_breakpoint':     LLOp(),
     'revdb_get_value':      LLOp(sideeffects=False),
     'revdb_get_unique_id':  LLOp(sideeffects=False),
     ## 'revdb_track_object':   LLOp(),
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -5,10 +5,11 @@
 CMD_QUIT     = -2
 CMD_FORWARD  = -3
 
-ANSWER_INIT    = -20
-ANSWER_STD     = -21
-ANSWER_FORKED  = -22
-ANSWER_AT_END  = -23
+ANSWER_INIT       = -20
+ANSWER_STD        = -21
+ANSWER_FORKED     = -22
+ANSWER_AT_END     = -23
+ANSWER_BREAKPOINT = -24
 
 
 class Message(object):
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -3,6 +3,11 @@
 from rpython.translator.revdb.message import *
 
 
+class Breakpoint(Exception):
+    def __init__(self, num):
+        self.num = num
+
+
 class ReplayProcess(object):
     """Represent one replaying subprocess.
 
@@ -73,8 +78,18 @@
     def forward(self, steps):
         """Move this subprocess forward in time."""
         self.send(Message(CMD_FORWARD, steps))
-        msg = self.expect(ANSWER_STD, Ellipsis, Ellipsis)
+        #
+        msg = self.recv()
+        if msg.cmd == ANSWER_BREAKPOINT:
+            bkpt_num = msg.arg3
+            msg = self.recv()
+        else:
+            bkpt_num = None
+        assert msg.cmd == ANSWER_STD
         self.update_times(msg)
+        #
+        if bkpt_num is not None:
+            raise Breakpoint(bkpt_num)
         return msg
 
 
@@ -83,7 +98,7 @@
     """
     MAX_SUBPROCESSES = 31       # maximum number of subprocesses
     STEP_RATIO = 0.25           # subprocess n is between subprocess n-1
-                                #   and the end, at this time fraction
+                                #   and the end, at this fraction of interval
 
     def __init__(self, executable, revdb_log_filename):
         s1, s2 = socket.socketpair()
@@ -120,7 +135,7 @@
             next_time = latest_done + int(self.STEP_RATIO * range_not_done) + 1
         return next_time
 
-    def forward(self, steps):
+    def go_forward(self, steps):
         """Go forward, for the given number of 'steps' of time.
 
         If needed, it will leave clones at intermediate times.
@@ -151,7 +166,9 @@
     def jump_in_time(self, target_time):
         """Jump in time at the given 'target_time'.
 
-        This function can close the active subprocess.
+        This function can close the active subprocess.  But you should
+        remove the breakpoints first, in case the same subprocess remains
+        active.
         """
         if target_time < 1:
             target_time = 1
@@ -161,8 +178,8 @@
         cur_time = self.get_current_time()
         if target_time >= cur_time:    # can go forward
             if cur_time >= max(self.paused):  # current time is past all forks
-                self.forward(target_time - cur_time)
+                self.go_forward(target_time - cur_time)
                 return
         # else, start from a fork
         self._resume(max(time for time in self.paused if time <= target_time))
-        self.forward(target_time - self.get_current_time())
+        self.go_forward(target_time - self.get_current_time())
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -196,10 +196,11 @@
 #define CMD_QUIT     (-2)
 #define CMD_FORWARD  (-3)
 
-#define ANSWER_INIT    (-20)
-#define ANSWER_STD     (-21)
-#define ANSWER_FORKED  (-22)
-#define ANSWER_AT_END  (-23)
+#define ANSWER_INIT       (-20)
+#define ANSWER_STD        (-21)
+#define ANSWER_FORKED     (-22)
+#define ANSWER_AT_END     (-23)
+#define ANSWER_BREAKPOINT (-24)
 
 typedef void (*rpy_revdb_command_fn)(rpy_revdb_command_t *, RPyString *);
 
@@ -640,27 +641,24 @@
         pending_after_forward = callback;
         break;
 
-    case 'k':       /* breakpoint */
-        if (time <= 0) {
-            fprintf(stderr, "revdb.breakpoint(): non-positive amount of "
-                            "steps\n");
-            exit(1);
-        }
-        if (stopped_time != 0) {
-            fprintf(stderr, "revdb.breakpoint(): cannot be called from a "
-                            "debug command\n");
-            exit(1);
-        }
-        rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + time;
-        pending_after_forward = callback;
-        break;
-
     default:
         abort();    /* unreachable */
     }
 }
 
 RPY_EXTERN
+void rpy_reverse_db_breakpoint(int64_t num)
+{
+    if (stopped_time != 0) {
+        fprintf(stderr, "revdb.breakpoint(): cannot be called from a "
+                        "debug command\n");
+        exit(1);
+    }
+    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
+    write_answer(ANSWER_BREAKPOINT, rpy_revdb.stop_point_break, 0, num);
+}
+
+RPY_EXTERN
 long long rpy_reverse_db_get_value(char value_id)
 {
     switch (value_id) {
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -87,6 +87,9 @@
 #define OP_REVDB_CHANGE_TIME(mode, time, callback, r)                   \
     rpy_reverse_db_change_time(mode, time, callback)
 
+#define OP_REVDB_BREAKPOINT(num, r)                                     \
+    rpy_reverse_db_breakpoint(num)
+
 #define OP_REVDB_GET_VALUE(value_id, r)                                 \
     r = rpy_reverse_db_get_value(value_id)
 
@@ -108,6 +111,7 @@
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
 RPY_EXTERN void rpy_reverse_db_change_time(char mode, long long time,
                                            void callback(void));
+RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
 RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id,
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -1,6 +1,8 @@
+import py
 from rpython.rlib import revdb
+from rpython.rlib.debug import debug_print
 from rpython.translator.revdb.message import *
-from rpython.translator.revdb.process import ReplayProcessGroup
+from rpython.translator.revdb.process import ReplayProcessGroup, Breakpoint
 
 from hypothesis import given, strategies
 
@@ -14,13 +16,24 @@
             pass
 
         class DBState:
-            pass
+            break_loop = -1
         dbstate = DBState()
 
+        def blip(cmd, extra):
+            debug_print('<<<', cmd.c_cmd, cmd.c_arg1,
+                               cmd.c_arg2, cmd.c_arg3, extra, '>>>')
+            if extra == 'set-breakpoint':
+                dbstate.break_loop = cmd.c_arg1
+            revdb.send_answer(42, cmd.c_cmd, -43, -44, extra)
+        lambda_blip = lambda: blip
+
         def main(argv):
+            revdb.register_debug_command(1, lambda_blip)
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
+                if dbstate.break_loop == i:
+                    revdb.breakpoint(99)
                 revdb.stop_point()
                 print op
             return 9
@@ -36,7 +49,7 @@
 
     def test_forward(self):
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.forward(100)
+        group.go_forward(100)
         assert group.get_current_time() == 10
         assert sorted(group.paused) == [1, 4, 6, 8, 9, 10]
         assert group._check_current_time(10)
@@ -47,3 +60,11 @@
         for target_time in target_times:
             group.jump_in_time(target_time)
             group._check_current_time(target_time)
+
+    def test_breakpoint(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        group.active.send(Message(1, 6, extra='set-breakpoint'))
+        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        e = py.test.raises(Breakpoint, group.go_forward, 10)
+        assert e.value.num == 99
+        group._check_current_time(7)

From pypy.commits at gmail.com  Mon Jun 20 05:18:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Jun 2016 02:18:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fixes stack computation error,
 adds a test. removed old argument calculation dating back to
 444ebc97d, 
Message-ID: <5767b4f6.571a1c0a.401a4.0410@mx.google.com>

Author: Richard Plangger 
Branch: py3.5
Changeset: r85234:3dd13008b96b
Date: 2016-06-20 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/3dd13008b96b/

Log:	fixes stack computation error, adds a test. removed old argument
	calculation dating back to 444ebc97d, the new behaviour is
	implemented, but the old calculation argument was provided to the
	byte code

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1133,11 +1133,6 @@
 
     def _make_call(self, n, # args already pushed
                    args, keywords):
-        #, starargs, kwargs
-        if args is not None:
-            arg = len(args) + n
-        else:
-            arg = n
         call_type = 0
         # the number of tuples and dictionaries on the stack
         nsubargs = 0
@@ -1207,11 +1202,11 @@
                     # Pack it all up
                     function_pos = n + (code & 1) + nkw + 1
                     self.emit_op_arg(ops.BUILD_MAP_UNPACK_WITH_CALL, (nsubkwargs | (function_pos << 8)))
-        
+
         assert n < 1<<8
         assert nkw < 1<<24
         n |= nkw << 8;
-        
+
         op = 0
         if call_type == 0:
             op = ops.CALL_FUNCTION
@@ -1221,7 +1216,7 @@
             op = ops.CALL_FUNCTION_KW
         elif call_type == 3:
             op = ops.CALL_FUNCTION_VAR_KW
-        self.emit_op_arg(op, arg)
+        self.emit_op_arg(op, n)
 
     def visit_Call(self, call):
         self.update_position(call.lineno)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1306,3 +1306,9 @@
             counts = self.count_instructions(source)
             assert ops.BUILD_SET not in counts
             assert ops.LOAD_CONST in counts
+
+    def test_call_function_var(self):
+        source = """call(*me)"""
+        code, blocks = generate_function_code(source, self.space)
+        # there is a stack computation error
+        assert blocks[0].instructions[3].arg == 0

From pypy.commits at gmail.com  Mon Jun 20 05:21:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 02:21:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix interact.py
Message-ID: <5767b5a3.692dc20a.489bc.7cf2@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85235:d761ca2cb60e
Date: 2016-06-20 11:22 +0200
http://bitbucket.org/pypy/pypy/changeset/d761ca2cb60e/

Log:	Fix interact.py

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -1,7 +1,7 @@
 import sys, re
 import subprocess, socket
 import traceback
-from rpython.translator.revdb.message import *
+from rpython.translator.revdb.process import ReplayProcessGroup, maxint64
 
 r_cmdline = re.compile(r"(\S+)\s*(.*)")
 
@@ -17,38 +17,17 @@
                 revdb_log_filename,))
         if executable is None:
             executable = fields[1]
-        #
-        s1, s2 = socket.socketpair()
-        subproc = subprocess.Popen(
-            [executable, '--revdb-replay', revdb_log_filename,
-             str(s2.fileno())])
-        s2.close()
-        self.subproc = subproc
-        child = ReplayProcess(subproc.pid, s1)
-        msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
-        self.total_stop_points = msg.arg2
-        msg = child.expect(ANSWER_STD, 1, Ellipsis)
-        child.update_times(msg)
-        self.active_child = child
-        self.paused_children = []
-        #
-        # fixed time division for now
-        if self.total_stop_points < 1:
-            raise ValueError("no stop points recorded in %r" % (
-                revdb_log_filename,))
-        self.fork_points = {1: child.clone()}
-        p = 1
-        while p < self.total_stop_points and len(self.fork_points) < 30:
-            p += int((self.total_stop_points - p) * 0.25) + 1
-            self.fork_points[p] = None
+        self.pgroup = ReplayProcessGroup(executable, revdb_log_filename)
 
     def interact(self):
         last_command = ''
         while True:
-            last_time = self.active_child.current_time
+            last_time = self.pgroup.get_current_time()
             prompt = '(%d)$ ' % last_time
+            sys.stdout.write(prompt)
+            sys.stdout.flush()
             try:
-                cmdline = raw_input(prompt).strip()
+                cmdline = raw_input().strip()
             except EOFError:
                 print
                 cmdline = 'quit'
@@ -57,6 +36,7 @@
             match = r_cmdline.match(cmdline)
             if not match:
                 continue
+            last_command = cmdline
             command, argument = match.groups()
             try:
                 runner = getattr(self, 'command_' + command)
@@ -67,7 +47,6 @@
                     runner(argument)
                 except Exception as e:
                     traceback.print_exc()
-                last_command = cmdline
 
     def command_help(self, argument):
         """Display commands summary"""
@@ -76,29 +55,42 @@
             if name.startswith('command_'):
                 command = name[len('command_'):]
                 docstring = getattr(self, name).__doc__ or 'undocumented'
-                print '\t%s\t%s' % (command, docstring)
+                print '\t%-12s %s' % (command, docstring)
 
     def command_quit(self, argument):
         """Exit the debugger"""
+        self.pgroup.close()
         sys.exit(0)
 
-    def change_child(self, target_time):
-        """If 'target_time' is not soon after 'self.active_child',
-        close it and fork another child."""
-        assert 1 <= target_time <= self.total_stop_points
-        best = max([key for key in self.fork_points if key <= target_time])
-        if best < self.active_child.current_time <= target_time:
-            pass     # keep 'active_child'
-        else:
-            self.active_child.close()
-            self.active_child = self.fork_points[best].clone()
+    def command_go(self, argument):
+        """Jump to time ARG"""
+        self.pgroup.jump_in_time(int(argument))
 
-    def command_go(self, argument):
-        """Go to time ARG"""
-        target_time = int(argument)
-        if target_time < 1:
-            target_time = 1
-        if target_time > self.total_stop_points:
-            target_time = self.total_stop_points
-        self.change_child(target_time)
-        self.active_child.forward(target_time - self.active_child.current_time)
+    def command_info(self, argument):
+        """Display various info ('info help' for more)"""
+        display = getattr(self, 'cmd_info_' + argument, self.cmd_info_help)
+        return display()
+
+    def cmd_info_help(self):
+        """Display info topics summary"""
+        print 'Available info topics:'
+        for name in dir(self):
+            if name.startswith('cmd_info_'):
+                command = name[len('cmd_info_'):]
+                docstring = getattr(self, name).__doc__ or 'undocumented'
+                print '\tinfo %-12s %s' % (command, docstring)
+
+    def cmd_info_paused(self):
+        """List current paused subprocesses"""
+        lst = [str(n) for n in sorted(self.pgroup.paused)]
+        print ', '.join(lst)
+
+    def command_step(self, argument):
+        """Run forward ARG steps (default 1)"""
+        self.pgroup.go_forward(int(argument or '1'))
+    command_s = command_step
+
+    def command_continue(self, argument):
+        """Run forward"""
+        self.pgroup.go_forward(maxint64)
+    command_c = command_continue
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -3,6 +3,9 @@
 from rpython.translator.revdb.message import *
 
 
+maxint64 = int(2**63 - 1)
+
+
 class Breakpoint(Exception):
     def __init__(self, num):
         self.num = num
@@ -183,3 +186,9 @@
         # else, start from a fork
         self._resume(max(time for time in self.paused if time <= target_time))
         self.go_forward(target_time - self.get_current_time())
+
+    def close(self):
+        """Close all subprocesses.
+        """
+        for subp in [self.active] + self.paused.values():
+            subp.close()
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -256,7 +256,7 @@
 
     def test_fork(self):
         child = self.replay()
-        child2 = child.fork()
+        child2 = child.clone()
         child.send(Message(CMD_FORWARD, 2))
         child.expect(ANSWER_STD, 3, Ellipsis)
         child2.send(Message(CMD_FORWARD, 1))
@@ -389,116 +389,3 @@
         child.expect(120, 3)
         child.send(Message(CMD_FORWARD, 0))
         child.expect(ANSWER_STD, 3, Ellipsis)
-
-    def test_get_unique_id_and_track_object(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r print-id')
-        child.expect(re.escape('<<>>\r\n')
-                     + r'obj.x=1002 (\d+) (\d+)'
-                     + re.escape('\r\n'
-                                 'blipped\r\n'
-                                 '(3)$ '))
-        object_id_3rd = int(child.match.group(1))
-        created_objects_3rd = int(child.match.group(2))
-        assert 0 < object_id_3rd < created_objects_3rd
-        #
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r print-id')
-        child.expect(re.escape('<<>>\r\n')
-                     + r'obj.x=1000 (\d+) (\d+)'
-                     + re.escape('\r\n'
-                                 'blipped\r\n'
-                                 '(1)$ '))
-        object_id_1st = int(child.match.group(1))
-        created_objects_1st = int(child.match.group(2))
-        assert 0 < object_id_1st < created_objects_1st
-        assert created_objects_1st <= object_id_3rd   # only created afterwards
-        #
-        child.sendline('r track-object %d' % object_id_3rd)
-        child.expectx('<<>>\r\n' % object_id_3rd +
-                      'blipped\r\n'
-                      '(1)$ ')
-        for i in [1, 2]:
-            child.sendline('r get-tracked-object')
-            child.expectx('<<>>\r\n'
-                          'none\r\n'
-                          'blipped\r\n'
-                          '(%d)$ ' % i)
-            child.sendline('__forward 1')
-            child.expectx('(%d)$ ' % (i + 1))
-        child.sendline('r get-tracked-object')
-        child.expectx('<<>>\r\n'
-                      'got obj.x=1002\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
-        child.sendline('__go 3')
-        child.expectx('(3)$ ')
-        child.sendline('r get-tracked-object')
-        child.expectx('<<>>\r\n'
-                      'none\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
-        #
-        child.sendline('__go 2')
-        child.expectx('(2)$ ')
-        child.sendline('r print-id')
-        child.expect(re.escape('<<>>\r\n')
-                     + r'obj.x=1001 (\d+) (\d+)'
-                     + re.escape('\r\n'
-                                 'blipped\r\n'
-                                 '(2)$ '))
-        object_id_2nd = int(child.match.group(1))
-        created_objects_2nd = int(child.match.group(2))
-        #
-        child.sendline('r track-object %d' % object_id_2nd)
-        child.expectx('<<>>\r\n' % object_id_2nd +
-                    'cannot track the creation of an object already created\r\n'
-                      'blipped\r\n'
-                      '(2)$ ')
-        child.sendline('r track-object 0')
-        child.expectx('<<>>\r\n'
-                      'cannot track a prebuilt or debugger-created object\r\n'
-                      'blipped\r\n'
-                      '(2)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r track-object %d' % object_id_2nd)
-        child.expectx('<<>>\r\n' % object_id_2nd +
-                      'blipped\r\n'
-                      '(1)$ ')
-        child.sendline('__forward 2')
-        child.expectx('(3)$ ')
-        child.sendline('r get-tracked-object')
-        child.expectx('<<>>\r\n'
-                      'got obj.x=1001\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
-        child.sendline('__forward 1')
-        child.expectx('At end.\r\n'
-                      '(3)$ ')
-        child.sendline('r get-tracked-object')
-        child.expectx('<<>>\r\n'
-                      'none\r\n'
-                      'blipped\r\n'
-                      '(3)$ ')
-
-    def test_first_created_uid(self):
-        child = self.replay()
-        child.expectx('(3)$ ')
-        child.sendline('r first-created-uid')
-        child.expectx('<<>>\r\n')
-        child.expect('first-created-uid=(\d+)\r\n')
-        first_created_id = int(child.match.group(1))
-        child.expectx('blipped\r\n'
-                      '(3)$ ')
-        child.sendline('__go 1')
-        child.expectx('(1)$ ')
-        child.sendline('r print-id')
-        child.expect(re.escape('<<>>\r\n')
-                     + r'obj.x=1000 (\d+) (\d+)'
-                     + re.escape('\r\n'
-                                 'blipped\r\n'
-                                 '(1)$ '))
-        assert int(child.match.group(2)) == first_created_id

From pypy.commits at gmail.com  Mon Jun 20 05:50:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 02:50:32 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <5767bc68.8aa6c20a.15839.54c5@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85236:e59f9c4f668a
Date: 2016-06-20 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/e59f9c4f668a/

Log:	in-progress

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -10,6 +10,11 @@
 from rpython.rtyper.lltypesystem import rffi
 
 
+CMD_PRINT = 1
+CMD_BACKTRACE = 2
+ANSWER_TEXT = 20
+
+
 def stop_point():
     """Indicates a point in the execution of the RPython program where
     the reverse-debugger can stop.  When reverse-debugging, we see
@@ -26,6 +31,9 @@
     """For RPython debug commands: writes an answer block to stdout"""
     llop.revdb_send_answer(lltype.Void, cmd, arg1, arg2, arg3, extra)
 
+def send_output(text):
+    send_answer(ANSWER_TEXT, extra=text)
+
 def current_time():
     """For RPython debug commands: returns the current time."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'c')
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -61,6 +61,7 @@
         """Exit the debugger"""
         self.pgroup.close()
         sys.exit(0)
+    command_q = command_quit
 
     def command_go(self, argument):
         """Jump to time ARG"""
@@ -87,10 +88,27 @@
 
     def command_step(self, argument):
         """Run forward ARG steps (default 1)"""
-        self.pgroup.go_forward(int(argument or '1'))
+        arg = int(argument or '1')
+        self.pgroup.go_forward(arg)
     command_s = command_step
 
+    def command_bstep(self, argument):
+        """Run backward ARG steps (default 1)"""
+        arg = int(argument or '1')
+        self.pgroup.jump_in_time(self.pgroup.get_current_time() - arg)
+    command_bs = command_bstep
+
     def command_continue(self, argument):
         """Run forward"""
         self.pgroup.go_forward(maxint64)
     command_c = command_continue
+
+    def command_print(self, argument):
+        """Print an expression"""
+        self.pgroup.print_cmd(argument)
+    command_p = command_print
+
+    def command_backtrace(self, argument):
+        """Show the backtrace"""
+        self.pgroup.show_backtrace()
+    command_bt = command_backtrace
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -4,6 +4,10 @@
 CMD_FORK     = -1
 CMD_QUIT     = -2
 CMD_FORWARD  = -3
+# extra commands which are not handled by revdb.c, but
+# by revdb.register_debug_command()
+CMD_PRINT     = 1
+CMD_BACKTRACE = 2
 
 ANSWER_INIT       = -20
 ANSWER_STD        = -21
@@ -11,6 +15,8 @@
 ANSWER_AT_END     = -23
 ANSWER_BREAKPOINT = -24
 
+ANSWER_TEXT       = 20
+
 
 class Message(object):
     """Represent messages sent and received to subprocesses
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -76,7 +76,10 @@
 
     def close(self):
         """Close this subprocess."""
-        self.send(Message(CMD_QUIT))
+        try:
+            self.send(Message(CMD_QUIT))
+        except socket.error:
+            pass
 
     def forward(self, steps):
         """Move this subprocess forward in time."""
@@ -95,6 +98,17 @@
             raise Breakpoint(bkpt_num)
         return msg
 
+    def print_text_answer(self):
+        while True:
+            msg = self.recv()
+            if msg.cmd == ANSWER_TEXT:
+                print msg.extra
+            elif msg.cmd == ANSWER_STD:
+                self.update_times(msg)
+                break
+            else:
+                print >> sys.stderr, "unexpected message %d" % (msg.cmd,)
+
 
 class ReplayProcessGroup(object):
     """Handle a family of subprocesses.
@@ -169,21 +183,12 @@
     def jump_in_time(self, target_time):
         """Jump in time at the given 'target_time'.
 
-        This function can close the active subprocess.  But you should
-        remove the breakpoints first, in case the same subprocess remains
-        active.
+        This function always closes the active subprocess.
         """
         if target_time < 1:
             target_time = 1
         if target_time > self.total_stop_points:
             target_time = self.total_stop_points
-
-        cur_time = self.get_current_time()
-        if target_time >= cur_time:    # can go forward
-            if cur_time >= max(self.paused):  # current time is past all forks
-                self.go_forward(target_time - cur_time)
-                return
-        # else, start from a fork
         self._resume(max(time for time in self.paused if time <= target_time))
         self.go_forward(target_time - self.get_current_time())
 
@@ -192,3 +197,15 @@
         """
         for subp in [self.active] + self.paused.values():
             subp.close()
+
+    def print_cmd(self, expression):
+        """Print an expression.
+        """
+        self.active.send(Message(CMD_PRINT, extra=expression))
+        self.active.print_text_answer()
+
+    def show_backtrace(self):
+        """Show the backtrace.
+        """
+        self.active.send(Message(CMD_BACKTRACE))
+        self.active.print_text_answer()
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -551,6 +551,7 @@
         s = make_rpy_string(cmd->extra_size);
         memcpy(_RPyString_AsString(s), extra, cmd->extra_size);
     }
+    pending_after_forward = &answer_std;
     execute_rpy_function(rpy_revdb_command_funcs[i], cmd, s);
 }
 

From pypy.commits at gmail.com  Mon Jun 20 06:08:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 03:08:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Reimplement 'tainting'
Message-ID: <5767c0a9.4a9bc20a.b68d3.2b97@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85237:ec6332290eb6
Date: 2016-06-20 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ec6332290eb6/

Log:	Reimplement 'tainting'

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -20,7 +20,7 @@
         self.pgroup = ReplayProcessGroup(executable, revdb_log_filename)
 
     def interact(self):
-        last_command = ''
+        last_command = 'help'
         while True:
             last_time = self.pgroup.get_current_time()
             prompt = '(%d)$ ' % last_time
@@ -65,7 +65,8 @@
 
     def command_go(self, argument):
         """Jump to time ARG"""
-        self.pgroup.jump_in_time(int(argument))
+        arg = int(argument or self.pgroup.get_current_time())
+        self.pgroup.jump_in_time(arg)
 
     def command_info(self, argument):
         """Display various info ('info help' for more)"""
@@ -89,7 +90,10 @@
     def command_step(self, argument):
         """Run forward ARG steps (default 1)"""
         arg = int(argument or '1')
-        self.pgroup.go_forward(arg)
+        if self.pgroup.is_tainted():
+            self.pgroup.jump_in_time(self.pgroup.get_current_time() + arg)
+        else:
+            self.pgroup.go_forward(arg)
     command_s = command_step
 
     def command_bstep(self, argument):
@@ -100,7 +104,7 @@
 
     def command_continue(self, argument):
         """Run forward"""
-        self.pgroup.go_forward(maxint64)
+        self.pgroup.jump_in_time(maxint64)
     command_c = command_continue
 
     def command_print(self, argument):
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -20,6 +20,7 @@
     def __init__(self, pid, control_socket):
         self.pid = pid
         self.control_socket = control_socket
+        self.tainted = False
 
     def _recv_all(self, size):
         pieces = []
@@ -83,6 +84,7 @@
 
     def forward(self, steps):
         """Move this subprocess forward in time."""
+        assert not self.tainted
         self.send(Message(CMD_FORWARD, steps))
         #
         msg = self.recv()
@@ -152,11 +154,15 @@
             next_time = latest_done + int(self.STEP_RATIO * range_not_done) + 1
         return next_time
 
+    def is_tainted(self):
+        return self.active.tainted
+
     def go_forward(self, steps):
         """Go forward, for the given number of 'steps' of time.
 
         If needed, it will leave clones at intermediate times.
-        Does not close the active subprocess.
+        Does not close the active subprocess.  Note that
+        is_tainted() must return false in order to use this.
         """
         assert steps >= 0
         while True:
@@ -201,6 +207,7 @@
     def print_cmd(self, expression):
         """Print an expression.
         """
+        self.active.tainted = True
         self.active.send(Message(CMD_PRINT, extra=expression))
         self.active.print_text_answer()
 

From pypy.commits at gmail.com  Mon Jun 20 06:17:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 03:17:58 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: locals.  fix tests.
Message-ID: <5767c2d6.4aa71c0a.c06a3.1c75@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85238:e0198f596958
Date: 2016-06-20 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/e0198f596958/

Log:	locals. fix tests.

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -10,9 +10,10 @@
 from rpython.rtyper.lltypesystem import rffi
 
 
-CMD_PRINT = 1
+CMD_PRINT     = 1
 CMD_BACKTRACE = 2
-ANSWER_TEXT = 20
+CMD_LOCALS    = 3
+ANSWER_TEXT   = 20
 
 
 def stop_point():
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -116,3 +116,7 @@
         """Show the backtrace"""
         self.pgroup.show_backtrace()
     command_bt = command_backtrace
+
+    def command_locals(self, argument):
+        """Show the locals"""
+        self.pgroup.show_locals()
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -8,6 +8,7 @@
 # by revdb.register_debug_command()
 CMD_PRINT     = 1
 CMD_BACKTRACE = 2
+CMD_LOCALS    = 3
 
 ANSWER_INIT       = -20
 ANSWER_STD        = -21
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -1,4 +1,4 @@
-import os, struct, socket, errno, subprocess
+import sys, os, struct, socket, errno, subprocess
 from rpython.translator.revdb import ancillary
 from rpython.translator.revdb.message import *
 
@@ -104,7 +104,8 @@
         while True:
             msg = self.recv()
             if msg.cmd == ANSWER_TEXT:
-                print msg.extra
+                sys.stdout.write(msg.extra)
+                sys.stdout.flush()
             elif msg.cmd == ANSWER_STD:
                 self.update_times(msg)
                 break
@@ -216,3 +217,9 @@
         """
         self.active.send(Message(CMD_BACKTRACE))
         self.active.print_text_answer()
+
+    def show_locals(self):
+        """Show the locals.
+        """
+        self.active.send(Message(CMD_LOCALS))
+        self.active.print_text_answer()
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -360,7 +360,6 @@
 
     def test_interaction_with_forward(self):
         child = self.replay()
-        child.send(Message(1, extra='oops'))
         child.send(Message(CMD_FORWARD, 50))
         child.expect(ANSWER_AT_END)
 
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -65,6 +65,7 @@
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
         group.active.send(Message(1, 6, extra='set-breakpoint'))
         group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.expect(ANSWER_STD, 1, Ellipsis)
         e = py.test.raises(Breakpoint, group.go_forward, 10)
         assert e.value.num == 99
         group._check_current_time(7)

From pypy.commits at gmail.com  Mon Jun 20 06:21:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Jun 2016 03:21:03 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: added micronumpy's division
 func to the test
Message-ID: <5767c38f.c5461c0a.a3aea.ffffa944@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85239:5450d2d80935
Date: 2016-06-20 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/5450d2d80935/

Log:	added micronumpy's division func to the test

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -75,9 +75,19 @@
                               type_system=self.type_system,
                               vec=vec, vec_all=vec_all)
 
+    @staticmethod
+    def rdiv(v1,v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+
     @given(data=st.data())
     @pytest.mark.parametrize('func', [lambda a,b: a+b,
-        lambda a,b: a*b, lambda a,b: a-b, lambda a,b: a / b])
+        lambda a,b: a*b, lambda a,b: a-b,
+        lambda a,b: VectorizeTests.rdiv(a,b)])
     def test_vector_simple_float(self, func, data):
         func = always_inline(func)
 

From pypy.commits at gmail.com  Mon Jun 20 06:21:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Jun 2016 03:21:06 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: catchup default
Message-ID: <5767c392.06a61c0a.43b5.ffffc7a2@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85240:ac310b1a036c
Date: 2016-06-20 12:20 +0200
http://bitbucket.org/pypy/pypy/changeset/ac310b1a036c/

Log:	catchup default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,10 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
 .. pull request #455
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 
@@ -31,3 +35,5 @@
 Simplify handling of interp-level tests and make it more forward-
 compatible.
 
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.1-alpha0"
-#define PYPY_VERSION_NUM  0x05030100
+#define PYPY_VERSION "5.3.2-alpha0"
+#define PYPY_VERSION_NUM  0x05030200
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -43,15 +43,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -195,7 +186,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -463,9 +454,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -508,12 +499,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -163,14 +163,10 @@
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
             guard_true(i32, descr=...)
-            i34 = getarrayitem_raw_i(#, #, descr=)  # XXX what are these?
-            guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i35 = getarrayitem_raw_i(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, 8)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, #, descr=...)                  # XXX
             jump(..., descr=...)
         """)
 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.unicodeobject import (
     decode_object, unicode_from_encoded_object,
     unicode_from_string, getdefaultencoding)
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
 class W_AbstractBytesObject(W_Root):
@@ -30,12 +31,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.str_w(self) is space.str_w(w_other)
+        s1 = space.str_w(self)
+        s2 = space.str_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.str_w(self)))
+        s = space.str_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ord(s[0])     # base values 0-255
+            else:
+                base = 256           # empty string: base value 256
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def unicode_w(self, space):
         # Use the default encoding.
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -6,6 +6,7 @@
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 from rpython.rlib.objectmodel import r_dict
 from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash
@@ -575,6 +576,23 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_FrozensetObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty frozensets are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty frozenset: base value 259
+        uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def _newobj(self, space, w_iterable):
         """Make a new frozenset by taking ownership of 'w_iterable'."""
         if type(self) is W_FrozensetObject:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -162,7 +162,8 @@
             buffer = _get_buffer(space, w_sub)
             res = count(value, buffer, start, end)
 
-        return space.wrap(max(res, 0))
+        assert res >= 0
+        return space.wrap(res)
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -211,6 +211,7 @@
 
         check(bytearray('abc').replace('b', bytearray('d')), 'adc')
         check(bytearray('abc').replace('b', 'd'), 'adc')
+        check(bytearray('').replace('a', 'ab'), '')
 
         check(bytearray('abc').upper(), 'ABC')
         check(bytearray('ABC').lower(), 'abc')
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -186,17 +186,36 @@
     def test_id_on_strs(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        u = u"a"
-        assert id(self.unwrap_wrap_unicode(u)) == id(u)
-        s = "a"
-        assert id(self.unwrap_wrap_str(s)) == id(s)
+        for u in [u"", u"a", u"aa"]:
+            assert id(self.unwrap_wrap_unicode(u)) == id(u)
+            s = str(u)
+            assert id(self.unwrap_wrap_str(s)) == id(s)
+        #
+        assert id('') == (256 << 4) | 11     # always
+        assert id(u'') == (257 << 4) | 11
+        assert id('a') == (ord('a') << 4) | 11
+        assert id(u'\u1234') == ((~0x1234) << 4) | 11
+
+    def test_id_of_tuples(self):
+        l = []
+        x = (l,)
+        assert id(x) != id((l,))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(()) == (258 << 4) | 11     # always
+
+    def test_id_of_frozensets(self):
+        x = frozenset([4])
+        assert id(x) != id(frozenset([4]))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(frozenset()) == (259 << 4) | 11     # always
+        assert id(frozenset([])) == (259 << 4) | 11   # always
 
     def test_identity_vs_id_primitives(self):
-        if self.cpython_apptest:
-            skip("cpython behaves differently")
         import sys
-        l = range(-10, 10)
-        for i in range(10):
+        l = range(-10, 10, 2)
+        for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
             l.append(long(i))
@@ -206,18 +225,15 @@
             l.append(i - 1j)
             l.append(1 + i * 1j)
             l.append(1 - i * 1j)
-            s = str(i)
-            l.append(s)
-            u = unicode(s)
-            l.append(u)
+            l.append((i,))
+            l.append(frozenset([i]))
         l.append(-0.0)
         l.append(None)
         l.append(True)
         l.append(False)
-        s = "s"
-        l.append(s)
-        s = u"s"
-        l.append(s)
+        l.append(())
+        l.append(tuple([]))
+        l.append(frozenset())
 
         for i, a in enumerate(l):
             for b in l[i:]:
@@ -228,21 +244,18 @@
     def test_identity_vs_id_str(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        import sys
-        l = range(-10, 10)
-        for i in range(10):
-            s = str(i)
+        l = []
+        def add(s, u):
             l.append(s)
             l.append(self.unwrap_wrap_str(s))
-            u = unicode(s)
+            l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
-        s = "s"
-        l.append(s)
-        l.append(self.unwrap_wrap_str(s))
-        s = u"s"
-        l.append(s)
-        l.append(self.unwrap_wrap_unicode(s))
+            l.append(u[:1] + u[1:])
+        for i in range(3, 18):
+            add(str(i), unicode(i))
+        add("s", u"s")
+        add("", u"")
 
         for i, a in enumerate(l):
             for b in l[i:]:
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -9,7 +9,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop,
     normalize_simple_slice)
-from pypy.objspace.std.util import negate
+from pypy.objspace.std.util import negate, IDTAG_SPECIAL, IDTAG_SHIFT
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import intmask
@@ -38,6 +38,23 @@
 class W_AbstractTupleObject(W_Root):
     __slots__ = ()
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractTupleObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty tuples are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty tuple: base value 258
+        uid = (258 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def __repr__(self):
         """representation for debugging purposes"""
         reprlist = [repr(w_item) for w_item in self.tolist()]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
@@ -52,12 +53,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.unicode_w(self) is space.unicode_w(w_other)
+        s1 = space.unicode_w(self)
+        s2 = space.unicode_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.unicode_w(self)))
+        s = space.unicode_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ~ord(s[0])      # negative base values
+            else:
+                base = 257       # empty unicode string: base value 257
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def str_w(self, space):
         return space.str_w(space.str(self))
diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py
--- a/pypy/objspace/std/util.py
+++ b/pypy/objspace/std/util.py
@@ -9,6 +9,12 @@
 IDTAG_FLOAT   = 5
 IDTAG_COMPLEX = 7
 IDTAG_METHOD  = 9
+IDTAG_SPECIAL = 11    # -1 - (-maxunicode-1): unichar
+                      # 0 - 255: char
+                      # 256: empty string
+                      # 257: empty unicode
+                      # 258: empty tuple
+                      # 259: empty frozenset
 
 CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
 BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,9 +1,9 @@
 # Edit these appropriately before running this script
 maj=5
 min=3
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -291,6 +291,7 @@
     return _search(value, other, start, end, SEARCH_COUNT)
 
 # -------------- substring searching helper ----------------
+# XXX a lot of code duplication with lltypesystem.rstr :-(
 
 SEARCH_COUNT = 0
 SEARCH_FIND = 1
@@ -309,6 +310,8 @@
     if end > len(value):
         end = len(value)
     if start > end:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     count = 0
@@ -326,6 +329,8 @@
     w = n - m
 
     if w < 0:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     mlast = m - 1
@@ -570,18 +575,20 @@
 
 class ByteListBuilder(object):
     def __init__(self, init_size=INIT_SIZE):
+        assert init_size >= 0
         self.l = newlist_hint(init_size)
 
     @specialize.argtype(1)
     def append(self, s):
+        l = self.l
         for c in s:
-            self.l.append(c)
+            l.append(c)
 
     @specialize.argtype(1)
     def append_slice(self, s, start, end):
-        assert 0 <= start <= end <= len(s)
-        for c in s[start:end]:
-            self.l.append(c)
+        l = self.l
+        for i in xrange(start, end):
+            l.append(s[i])
 
     def append_multiple_char(self, c, times):
         assert isinstance(c, str)
@@ -589,8 +596,9 @@
 
     def append_charpsize(self, s, size):
         assert size >= 0
+        l = self.l
         for i in xrange(size):
-            self.l.append(s[i])
+            l.append(s[i])
 
     def build(self):
         return self.l
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -231,6 +231,10 @@
     check_search(count, 'one two three', 'e', 0, 1, res=0)
     check_search(count, 'one two three', '', 0, 13, res=14)
 
+    check_search(count, '', 'ab', 0, 0, res=0)
+    check_search(count, 'a', 'ab', 0, 1, res=0)
+    check_search(count, 'ac', 'ab', 0, 2, res=0)
+
 
 class TestTranslates(BaseRtypingTest):
     def test_split_rsplit(self):
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -972,6 +972,13 @@
             s.count(s, -10)
         py.test.raises(AnnotatorError, self.interpret, f, ())
 
+    def test_count_in_empty_string(self):
+        const = self.const
+        def fn():
+            return const('').count(const('ab'))
+        res = self.interpret(fn, [])
+        assert res == 0
+
     def test_getitem_exc(self):
         const = self.const
         def f(x):

From pypy.commits at gmail.com  Mon Jun 20 06:59:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 03:59:52 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Breakpoints
Message-ID: <5767cca8.06a81c0a.7a8c4.2a64@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85241:813b058bf361
Date: 2016-06-20 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/813b058bf361/

Log:	Breakpoints

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -10,10 +10,11 @@
 from rpython.rtyper.lltypesystem import rffi
 
 
-CMD_PRINT     = 1
-CMD_BACKTRACE = 2
-CMD_LOCALS    = 3
-ANSWER_TEXT   = 20
+CMD_PRINT       = 1
+CMD_BACKTRACE   = 2
+CMD_LOCALS      = 3
+CMD_BREAKPOINTS = 4
+ANSWER_TEXT     = 20
 
 
 def stop_point():
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -2,6 +2,7 @@
 import subprocess, socket
 import traceback
 from rpython.translator.revdb.process import ReplayProcessGroup, maxint64
+from rpython.translator.revdb.process import Breakpoint
 
 r_cmdline = re.compile(r"(\S+)\s*(.*)")
 
@@ -91,11 +92,17 @@
         """Run forward ARG steps (default 1)"""
         arg = int(argument or '1')
         if self.pgroup.is_tainted():
-            self.pgroup.jump_in_time(self.pgroup.get_current_time() + arg)
-        else:
-            self.pgroup.go_forward(arg)
+            self.pgroup.jump_in_time(self.pgroup.get_current_time())
+            assert not self.pgroup.is_tainted()
+        self.move_forward(arg)
     command_s = command_step
 
+    def move_forward(self, steps):
+        try:
+            self.pgroup.go_forward(steps)
+        except Breakpoint as b:
+            print 'Hit breakpoint %d' % (b.num,)
+
     def command_bstep(self, argument):
         """Run backward ARG steps (default 1)"""
         arg = int(argument or '1')
@@ -104,7 +111,8 @@
 
     def command_continue(self, argument):
         """Run forward"""
-        self.pgroup.jump_in_time(maxint64)
+        self.move_forward(self.pgroup.get_max_time() -
+                          self.pgroup.get_current_time())
     command_c = command_continue
 
     def command_print(self, argument):
@@ -120,3 +128,21 @@
     def command_locals(self, argument):
         """Show the locals"""
         self.pgroup.show_locals()
+
+    def command_break(self, argument):
+        """Add a breakpoint"""
+        new = 1
+        while new in self.pgroup.breakpoints:
+            new += 1
+        self.pgroup.breakpoints[new] = argument
+        print "Breakpoint %d added" % (new,)
+    command_b = command_break
+
+    def command_delete(self, argument):
+        """Delete a breakpoint"""
+        arg = int(argument)
+        if arg not in self.pgroup.breakpoints:
+            print "No breakpoint number %d" % (new,)
+        else:
+            del self.pgroup.breakpoints[arg]
+            print "Breakpoint %d deleted" % (new,)
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -6,9 +6,10 @@
 CMD_FORWARD  = -3
 # extra commands which are not handled by revdb.c, but
 # by revdb.register_debug_command()
-CMD_PRINT     = 1
-CMD_BACKTRACE = 2
-CMD_LOCALS    = 3
+CMD_PRINT       = 1
+CMD_BACKTRACE   = 2
+CMD_LOCALS      = 3
+CMD_BREAKPOINTS = 4
 
 ANSWER_INIT       = -20
 ANSWER_STD        = -21
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -1,4 +1,5 @@
 import sys, os, struct, socket, errno, subprocess
+from contextlib import contextmanager
 from rpython.translator.revdb import ancillary
 from rpython.translator.revdb.message import *
 
@@ -17,10 +18,11 @@
     It can be either the one started with --revdb-replay, or a fork.
     """
 
-    def __init__(self, pid, control_socket):
+    def __init__(self, pid, control_socket, breakpoints_cache={}):
         self.pid = pid
         self.control_socket = control_socket
         self.tainted = False
+        self.breakpoints_cache = breakpoints_cache    # don't change this dict
 
     def _recv_all(self, size):
         pieces = []
@@ -56,6 +58,10 @@
             assert msg.extra == extra
         return msg
 
+    def expect_std(self):
+        msg = self.expect(ANSWER_STD, Ellipsis, Ellipsis)
+        self.update_times(msg)
+
     def update_times(self, msg):
         self.current_time = msg.arg1
         self.currently_created_objects = msg.arg2
@@ -71,7 +77,8 @@
         msg = self.expect(ANSWER_FORKED, Ellipsis, Ellipsis, Ellipsis)
         self.update_times(msg)
         child_pid = msg.arg3
-        other = ReplayProcess(child_pid, s1)
+        other = ReplayProcess(child_pid, s1,
+                              breakpoints_cache=self.breakpoints_cache)
         other.update_times(msg)
         return other
 
@@ -129,11 +136,11 @@
         child = ReplayProcess(initial_subproc.pid, s1)
         msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
         self.total_stop_points = msg.arg2
-        msg = child.expect(ANSWER_STD, 1, Ellipsis)
-        child.update_times(msg)
+        child.expect_std()
 
         self.active = child
         self.paused = {1: child.clone()}     # {time: subprocess}
+        self.breakpoints = {}
 
     def get_current_time(self):
         return self.active.current_time
@@ -166,6 +173,7 @@
         is_tainted() must return false in order to use this.
         """
         assert steps >= 0
+        self.update_breakpoints()
         while True:
             cur_time = self.get_current_time()
             if cur_time + steps > self.total_stop_points:
@@ -182,6 +190,19 @@
             self.paused[clone.current_time] = clone
         self.active.forward(steps)
 
+    def update_breakpoints(self):
+        if self.active.breakpoints_cache != self.breakpoints:
+            if self.breakpoints:
+                breakpoints = [self.breakpoints.get(n, '')
+                               for n in range(max(self.breakpoints) + 1)]
+            else:
+                breakpoints = []
+            extra = '\x00'.join(breakpoints)
+            self.active.breakpoints_cache = None
+            self.active.send(Message(CMD_BREAKPOINTS, extra=extra))
+            self.active.expect_std()
+            self.active.breakpoints_cache = self.breakpoints.copy()
+
     def _resume(self, from_time):
         clone_me = self.paused[from_time]
         self.active.close()
@@ -197,7 +218,17 @@
         if target_time > self.total_stop_points:
             target_time = self.total_stop_points
         self._resume(max(time for time in self.paused if time <= target_time))
-        self.go_forward(target_time - self.get_current_time())
+        with self._breakpoints_disabled():
+            self.go_forward(target_time - self.get_current_time())
+
+    @contextmanager
+    def _breakpoints_disabled(self):
+        old =  self.breakpoints
+        self.breakpoints = {}
+        try:
+            yield
+        finally:
+            self.breakpoints = old
 
     def close(self):
         """Close all subprocesses.

From pypy.commits at gmail.com  Mon Jun 20 07:00:38 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:00:38 -0700 (PDT)
Subject: [pypy-commit] pypy stmgc-c8: fix source of conflicts within
 mapdict_cache
Message-ID: <5767ccd6.4a9bc20a.b68d3.4228@mx.google.com>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r85242:1352e325ed85
Date: 2016-05-27 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/1352e325ed85/

Log:	fix source of conflicts within mapdict_cache

	Obviously, changing a CacheEntry still causes conflicts. For now,
	create new entries every time, but that may be fixable by making
	CacheEntry noconflict too.

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -944,6 +944,7 @@
             if version_tag is self.version_tag:
                 # everything matches, it's incredibly fast
                 if map.space.config.objspace.std.withmethodcachecounter:
+                    assert not map.space.config.translation.stm
                     self.success_counter += 1
                 return True
         return False
@@ -966,7 +967,7 @@
     # pycode._mapdict_caches = [INVALID_CACHE_ENTRY] * num_entries
     pycode._mapdict_caches = NULL_MAPDICTCACHE
 
-def lazy_init_mapdict_cache(pycode):
+def lazy_alloc_mapdict_cache(pycode):
     assert we_are_translated()
     num_entries = len(pycode.co_names_w)
     if pycode.space.config.translation.stm:
@@ -991,10 +992,11 @@
     if not we_are_translated():
         return
     if pycode._mapdict_caches is NULL_MAPDICTCACHE:
-        lazy_init_mapdict_cache(pycode)
+        lazy_alloc_mapdict_cache(pycode)
     #
     entry = annlowlevel.cast_gcref_to_instance(CacheEntry, pycode._mapdict_caches[nameindex])
-    if entry is pycode._mapdict_cache_invalid:
+    if entry is pycode._mapdict_cache_invalid or pycode.space.config.translation.stm:
+        # always a new entry for STM (CacheEntry is not no-conflict (XXX?))
         entry = CacheEntry()
         pycode._mapdict_caches[nameindex] = annlowlevel.cast_instance_to_gcref(entry)
     entry.map_wref = weakref.ref(map)
@@ -1010,7 +1012,7 @@
     if not we_are_translated():
         return LOAD_ATTR_slowpath(pycode, w_obj, nameindex, w_obj._get_mapdict_map())
     if pycode._mapdict_caches is NULL_MAPDICTCACHE:
-        lazy_init_mapdict_cache(pycode)
+        lazy_alloc_mapdict_cache(pycode)
     #
     entry = annlowlevel.cast_gcref_to_instance(CacheEntry, pycode._mapdict_caches[nameindex])
     map = w_obj._get_mapdict_map()
@@ -1073,7 +1075,7 @@
     if not we_are_translated():
         return False
     if pycode._mapdict_caches is NULL_MAPDICTCACHE:
-        lazy_init_mapdict_cache(pycode)
+        lazy_alloc_mapdict_cache(pycode)
     #
     entry = annlowlevel.cast_gcref_to_instance(CacheEntry, pycode._mapdict_caches[nameindex])
     if entry.is_valid_for_obj(w_obj):

From pypy.commits at gmail.com  Mon Jun 20 07:00:40 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:00:40 -0700 (PDT)
Subject: [pypy-commit] pypy stmgc-c8: fix become_inevitable in vmprof
Message-ID: <5767ccd8.4aa6c20a.8b349.ffff9089@mx.google.com>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r85243:e7a5293758ae
Date: 2016-05-27 10:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e7a5293758ae/

Log:	fix become_inevitable in vmprof

	Already OK in the JIT, but the RPython part caused inevitable
	transactions by accessing the raw memory. Inevitable is not
	necessary in this case as we reset the vmprof_tl_stack on abort
	(stm/extracode.h)

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -72,7 +72,8 @@
 VMPROFSTACK.become(rffi.CStruct("vmprof_stack_s",
                                 ('next', PVMPROFSTACK),
                                 ('value', lltype.Signed),
-                                ('kind', lltype.Signed)))
+                                ('kind', lltype.Signed),
+                                hints={'stm_dont_track_raw_accesses': True}))
 # ----------
 
 

From pypy.commits at gmail.com  Mon Jun 20 07:00:42 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:00:42 -0700 (PDT)
Subject: [pypy-commit] pypy stmgc-c8: Merge
Message-ID: <5767ccda.8438c20a.7991b.0557@mx.google.com>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r85244:477eccd1cf3e
Date: 2016-06-20 12:59 +0200
http://bitbucket.org/pypy/pypy/changeset/477eccd1cf3e/

Log:	Merge

diff --git a/rpython/tool/perf-disassemble.sh b/rpython/tool/perf-disassemble.sh
--- a/rpython/tool/perf-disassemble.sh
+++ b/rpython/tool/perf-disassemble.sh
@@ -12,10 +12,12 @@
 # $ dolphin-emu -P /tmp -b -e $game
 # $ perf top -p $(pidof dolphin-emu) --objdump ./Tools/perf-disassemble.sh
 
-flavor=att
-raw=r
+flavor=intel
+#raw=r
+raw=
 src=
 
+echo $@ > ~/bla
 
 [[ "${@: -1}" != /tmp/perf-*.map ]] && { objdump "$@"; exit; }
 
@@ -47,4 +49,4 @@
             ;;
     esac
 done
-gdb -q -p $pid -ex "set disassembly $flavor" -ex "disas /$raw$src $start,$stop" -ex q -batch
+gdb -q -p $pid -ex "set disassembly $flavor" -ex "disas /$raw$src $start,$stop" -ex q -batch | sed "s/=>/  /g"

From pypy.commits at gmail.com  Mon Jun 20 07:04:34 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:34 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add another benchmark
 runner
Message-ID: <5767cdc2.2523c20a.e0232.ffff9ffc@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r352:22b1aafdeea4
Date: 2016-05-26 10:39 +0200
http://bitbucket.org/pypy/benchmarks/changeset/22b1aafdeea4/

Log:	add another benchmark runner

diff --git a/multithread/runner.py b/multithread/runner.py
new file mode 100755
--- /dev/null
+++ b/multithread/runner.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+""" Usage: runner.py  
+"""
+
+config = {
+    "defaults": {
+        # global defaults:
+        "file": None,
+        "threads": [1, 2, 4, 8,],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "args": [],
+        "cwd": ".", # relative to file
+        "PYTHONPATH": ".",
+    },
+
+    "benchs": {
+        # list of benchmarks:
+        "raytrace": {
+            "file": "raytrace/raytrace.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 5,
+            "warmiters": 5,
+            "args": ["1024", "1024"] # w, h
+        },
+    },
+}
+
+
+import json
+import time
+import os, sys
+import copy
+import pprint
+from subprocess import Popen, PIPE
+
+
+def run_benchmark(python_exec, bench_config):
+    vmstarts = bench_config['vmstarts']
+    threads = bench_config['threads']
+    print "## run_benchmark", bench_config['file']
+
+    failures = []
+    timings = []
+    for ts in threads:
+        for vm in range(vmstarts):
+            print "threads: %s, vm: %s" % (ts, vm)
+
+            bench_file = os.path.abspath(bench_config['file'])
+            cmd = ([python_exec,
+                    bench_file,
+                    str(bench_config['warmiters']),
+                    str(ts)]
+                   + bench_config['args'])
+            cmd_str = " ".join(cmd)
+
+            cwd, _ = os.path.split(bench_file)
+            cwd = os.path.join(cwd, bench_config['cwd'])
+            env = os.environ.copy()
+            env['PYTHONPATH'] = bench_config['PYTHONPATH']
+            print "running:", cmd_str, "in", cwd, "with PYTHONPATH=", env['PYTHONPATH']
+
+            try:
+                print env['PYTHONPATH']
+                p = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd)
+                if p.wait() != 0:
+                    # error
+                    stdout, stderr = p.stdout.read(), p.stderr.read()
+                    failure = {
+                        'cmd': cmd_str,
+                        'stdout': stdout,
+                        'stderr': stderr,
+                    }
+                    failures.append(failure)
+                    print "failure:", failure
+                else:
+                    stdout, stderr = p.stdout.read(), p.stderr.read()
+                    iter_times = extract_iter_times(stdout)
+                    times = {
+                        'cmd': " ".join(cmd),
+                        'threads': ts,
+                        'vmstarts': vm,
+                        'stdout': stdout,
+                        'stderr': stderr,
+                        'warmiters': iter_times,
+                    }
+                    timings.append(times)
+                    print "timing:", times
+            finally:
+                pass
+    return failures, timings
+
+
+def run_benchmarks(results):
+    all_results = results['results'] = {}
+    all_config = results['config']
+    for bench_key, temp_config in all_config['benchs'].items():
+        # load global defaults and overwrite with bench-specific config:
+        bench_config = copy.deepcopy(all_config['defaults'])
+        bench_config.update(temp_config)
+
+        try:
+            failures, timings = run_benchmark(
+                results['python'], bench_config)
+        except Exception as e:
+            all_results[bench_key] = {
+                'fail_reason': str(e)}
+        else:
+            all_results[bench_key] = {
+                'failures': failures,
+                'timings': timings}
+
+        print bench_key, bench_config
+
+
+
+
+def main(argv):
+    result_file = argv[0]
+    python_exec = argv[1]
+
+    if os.path.exists(result_file):
+        with open(result_file, 'r+') as f:
+            results = json.loads(f.read())
+    else:
+        results = {}
+
+    run_key = time.ctime()
+    results[run_key] = {'config': config,
+                        'python': python_exec}
+    try:
+        run_benchmarks(results[run_key])
+        print results[run_key]['results']
+    finally:
+        with open(result_file, 'w') as f:
+            f.write(json.dumps(
+                results, sort_keys=True,
+                indent=4, separators=(',', ': ')))
+
+
+if __name__ != '__main__': #FIXME: emacs bug?
+    main(sys.argv[1:])
+else:
+    main(["results.json", "/usr/bin/python"])

From pypy.commits at gmail.com  Mon Jun 20 07:04:36 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:36 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add timeout and
 config-as-parameter
Message-ID: <5767cdc4.07ecc20a.13dc0.ffff98e3@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r353:2b8918b59cb7
Date: 2016-05-27 09:34 +0200
http://bitbucket.org/pypy/benchmarks/changeset/2b8918b59cb7/

Log:	add timeout and config-as-parameter

diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -5,7 +5,7 @@
 try:
     from pypystm import atomic, getsegmentlimit, hint_commit_soon
 except ImportError:
-    raise
+    print "NON-STM EXECUTION"
     atomic = RLock()
     def getsegmentlimit():
         return 1
@@ -144,9 +144,9 @@
     def _task(self, func, *args, **kwargs):
         with self._cond:
             try:
-                hint_commit_soon()
+                #hint_commit_soon()
                 self._result = func(*args, **kwargs)
-                hint_commit_soon()
+                #hint_commit_soon()
             except Exception as e:
                 self._exception = e
             finally:
@@ -173,10 +173,10 @@
     def _task(self, func, *args, **kwargs):
         with self._cond:
             try:
-                hint_commit_soon()
+                #hint_commit_soon()
                 with atomic:
                     self._result = func(*args, **kwargs)
-                hint_commit_soon()
+                #hint_commit_soon()
             except Exception as e:
                 self._exception = e
             finally:
diff --git a/multithread/config-raytrace.json b/multithread/config-raytrace.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-raytrace.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "raytrace": {
+            "file": "raytrace/raytrace.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 5,
+            "warmiters": 3,
+            "args": ["512", "4096"]
+        }
+    }
+}
diff --git a/multithread/raytrace/raytrace.py b/multithread/raytrace/raytrace.py
--- a/multithread/raytrace/raytrace.py
+++ b/multithread/raytrace/raytrace.py
@@ -136,8 +136,7 @@
 
 
 
-def task(img, x, h, cameraPos, objs, lightSource):
-    line = img[x]
+def task(line, x, h, cameraPos, objs, lightSource):
     for y in range(h):
         with atomic:
             ray = Ray(cameraPos,
@@ -173,10 +172,10 @@
         img.append([0.0] * h)
     parallel_time = time.time()
     for x in range(w):
-        future_dispatcher(ths, img, x, h, cameraPos, objs, lightSource)
+        future_dispatcher(ths, img[x], x, h, cameraPos, objs, lightSource)
 
     for f in futures:
-        print f()
+        f()
     del futures[:]
     parallel_time = time.time() - parallel_time
 
@@ -185,6 +184,29 @@
     return parallel_time
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    w, h = int(argv[2]), int(argv[3])
+
+    print "params (iters, threads, w, h):", warmiters, threads, w, h
+
+    print "do warmup:"
+    for i in range(5):
+        print "iter", i, "time:", run(threads, w, h)
+
+    print "turn off jit"
+    import pypyjit, gc
+    pypyjit.set_param("off")
+    pypyjit.set_param("threshold=999999999,trace_eagerness=99999999")
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        times.append(run(threads, w, h))
+    print "warmiters:", times
 
 if __name__ == '__main__':
-    run()
+    import sys
+    main(sys.argv[1:])
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -1,32 +1,7 @@
 #!/usr/bin/env python
-""" Usage: runner.py  
+""" Usage: runner.py   
 """
 
-config = {
-    "defaults": {
-        # global defaults:
-        "file": None,
-        "threads": [1, 2, 4, 8,],
-        "vmstarts": 3,
-        "warmiters": 5,
-        "args": [],
-        "cwd": ".", # relative to file
-        "PYTHONPATH": ".",
-    },
-
-    "benchs": {
-        # list of benchmarks:
-        "raytrace": {
-            "file": "raytrace/raytrace.py",
-            "PYTHONPATH": "..",
-            "vmstarts": 5,
-            "warmiters": 5,
-            "args": ["1024", "1024"] # w, h
-        },
-    },
-}
-
-
 import json
 import time
 import os, sys
@@ -35,10 +10,18 @@
 from subprocess import Popen, PIPE
 
 
+def extract_iter_times(stdout):
+    for line in stdout.split('\n'):
+        if "warmiters" in line:
+            # warmiters: [1.2,3.1,]
+            times = line.split(':')[1].strip()[1:-1]
+            return [float(t) for t in times.split(',')]
+    return None
+
 def run_benchmark(python_exec, bench_config):
     vmstarts = bench_config['vmstarts']
     threads = bench_config['threads']
-    print "## run_benchmark", bench_config['file']
+    print "## run_benchmark", bench_config
 
     failures = []
     timings = []
@@ -58,16 +41,24 @@
             cwd = os.path.join(cwd, bench_config['cwd'])
             env = os.environ.copy()
             env['PYTHONPATH'] = bench_config['PYTHONPATH']
-            print "running:", cmd_str, "in", cwd, "with PYTHONPATH=", env['PYTHONPATH']
+            print "running:", cmd_str
 
             try:
-                print env['PYTHONPATH']
                 p = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd)
+                # XXX: process could deadlock if stdout pipe is full -> never terminate -> timeout
+                start_time = time.time()
+                while p.poll() is None:
+                    time.sleep(0.5)
+                    if time.time() - start_time > 30 * 60:
+                        # kill after 30min
+                        p.kill()
+
                 if p.wait() != 0:
                     # error
                     stdout, stderr = p.stdout.read(), p.stderr.read()
                     failure = {
                         'cmd': cmd_str,
+                        'exitcode': p.returncode,
                         'stdout': stdout,
                         'stderr': stderr,
                     }
@@ -75,6 +66,7 @@
                     print "failure:", failure
                 else:
                     stdout, stderr = p.stdout.read(), p.stderr.read()
+                    print stdout
                     iter_times = extract_iter_times(stdout)
                     times = {
                         'cmd': " ".join(cmd),
@@ -85,9 +77,14 @@
                         'warmiters': iter_times,
                     }
                     timings.append(times)
-                    print "timing:", times
-            finally:
-                pass
+                    print "warmiters:", times['warmiters']
+            except Exception as e:
+                failures.append({
+                    'cmd': cmd_str, 'exception': str(e)})
+            except KeyboardInterrupt as e:
+                failures.append({
+                    'cmd': cmd_str, 'exception': str(e)})
+                return failures, timings
     return failures, timings
 
 
@@ -116,8 +113,14 @@
 
 
 def main(argv):
-    result_file = argv[0]
-    python_exec = argv[1]
+    """ Usage: runner.py    """
+    python_exec = argv[0]
+    config_file = argv[1]
+    result_file = argv[2]
+
+    assert os.path.exists(config_file)
+    with open(config_file, 'r') as f:
+        config = json.loads(f.read())
 
     if os.path.exists(result_file):
         with open(result_file, 'r+') as f:
@@ -141,4 +144,4 @@
 if __name__ != '__main__': #FIXME: emacs bug?
     main(sys.argv[1:])
 else:
-    main(["results.json", "/usr/bin/python"])
+    main(["pypy-c", "config-raytrace.json", "results.json",])

From pypy.commits at gmail.com  Mon Jun 20 07:04:38 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:38 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: progress
Message-ID: <5767cdc6.09f6c20a.38379.ffff9139@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r354:ea22d0e33d51
Date: 2016-05-27 13:00 +0200
http://bitbucket.org/pypy/benchmarks/changeset/ea22d0e33d51/

Log:	progress

diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -15,6 +15,16 @@
 def print_abort_info(tm=0.0):
     "backward compatibility: no-op"
 
+def turn_jitting_off():
+    """function that turns off the JIT
+    shouldn't be in this module, but well..."""
+    try:
+        import pypyjit
+        pypyjit.set_param("off")
+        pypyjit.set_param("threshold=999999999,trace_eagerness=99999999")
+    except ImportError:
+        pass
+
 
 class TLQueue_concurrent(object):
     def __init__(self):
@@ -144,9 +154,9 @@
     def _task(self, func, *args, **kwargs):
         with self._cond:
             try:
-                #hint_commit_soon()
+                hint_commit_soon()
                 self._result = func(*args, **kwargs)
-                #hint_commit_soon()
+                hint_commit_soon()
             except Exception as e:
                 self._exception = e
             finally:
@@ -173,10 +183,10 @@
     def _task(self, func, *args, **kwargs):
         with self._cond:
             try:
-                #hint_commit_soon()
+                hint_commit_soon()
                 with atomic:
                     self._result = func(*args, **kwargs)
-                #hint_commit_soon()
+                hint_commit_soon()
             except Exception as e:
                 self._exception = e
             finally:
diff --git a/multithread/config-raytrace.json b/multithread/config-raytrace.json
--- a/multithread/config-raytrace.json
+++ b/multithread/config-raytrace.json
@@ -14,8 +14,8 @@
             "file": "raytrace/raytrace.py",
             "PYTHONPATH": "..",
             "vmstarts": 5,
-            "warmiters": 3,
-            "args": ["512", "4096"]
+            "warmiters": 2,
+            "args": ["128", "512"]
         }
     }
 }
diff --git a/multithread/raytrace/raytrace.py b/multithread/raytrace/raytrace.py
--- a/multithread/raytrace/raytrace.py
+++ b/multithread/raytrace/raytrace.py
@@ -4,7 +4,7 @@
 from math import sqrt, pi
 from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
-    print_abort_info, hint_commit_soon)
+    print_abort_info, hint_commit_soon, turn_jitting_off)
 import time
 
 #import platform
@@ -138,7 +138,7 @@
 
 def task(line, x, h, cameraPos, objs, lightSource):
     for y in range(h):
-        with atomic:
+        if 1:#with atomic:
             ray = Ray(cameraPos,
                       (Vector(x/50.0-5,y/50.0-5,0)-cameraPos).normal())
             col = trace(ray, objs, lightSource, 10)
@@ -193,13 +193,12 @@
     print "params (iters, threads, w, h):", warmiters, threads, w, h
 
     print "do warmup:"
-    for i in range(5):
+    for i in range(3):
         print "iter", i, "time:", run(threads, w, h)
 
-    print "turn off jit"
-    import pypyjit, gc
-    pypyjit.set_param("off")
-    pypyjit.set_param("threshold=999999999,trace_eagerness=99999999")
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
     print "do", warmiters, "real iters:"
     times = []
     for i in range(warmiters):
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -81,9 +81,9 @@
             except Exception as e:
                 failures.append({
                     'cmd': cmd_str, 'exception': str(e)})
-            except KeyboardInterrupt as e:
+            except KeyboardInterrupt:
                 failures.append({
-                    'cmd': cmd_str, 'exception': str(e)})
+                    'cmd': cmd_str, 'exception': 'KeyboardInterrupt'})
                 return failures, timings
     return failures, timings
 
@@ -128,9 +128,24 @@
     else:
         results = {}
 
+    p = Popen([python_exec, "--version"],
+              stdout=PIPE, stderr=PIPE)
+    _, python_version = p.communicate()
+    assert p.returncode == 0
+    print python_version
+
+    p = Popen(["hg", "id"], stdout=PIPE, stderr=PIPE)
+    hg_id, _ = p.communicate()
+    assert p.returncode == 0
+    print "id", hg_id
+
     run_key = time.ctime()
-    results[run_key] = {'config': config,
-                        'python': python_exec}
+    results[run_key] = {
+        'config': config,
+        'python': python_exec,
+        'python-version': python_version,
+        'hg-id': hg_id}
+
     try:
         run_benchmarks(results[run_key])
         print results[run_key]['results']

From pypy.commits at gmail.com  Mon Jun 20 07:04:39 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:39 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add some configs,
 fix some benchmarks
Message-ID: <5767cdc7.24f9c20a.1929.ffffbf72@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r355:47e822fcaecf
Date: 2016-05-27 15:55 +0200
http://bitbucket.org/pypy/benchmarks/changeset/47e822fcaecf/

Log:	add some configs, fix some benchmarks

diff --git a/multithread/btree/btree.py b/multithread/btree/btree.py
--- a/multithread/btree/btree.py
+++ b/multithread/btree/btree.py
@@ -2,7 +2,7 @@
 
 from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
-    hint_commit_soon, print_abort_info)
+    hint_commit_soon, turn_jitting_off)
 import time, threading
 
 import random
@@ -205,7 +205,7 @@
             ancestors.append((node, index))
         node, index = ancestors.pop()
         node.insert(index, item, ancestors)
-        hint_commit_soon()
+        #hint_commit_soon()
         return True
 
     def remove(self, item):
@@ -214,7 +214,7 @@
         if self._present(item, ancestors):
             node, index = ancestors.pop()
             node.remove(index, ancestors)
-            hint_commit_soon()
+            #hint_commit_soon()
         # else:
         #     raise ValueError("%r not in %s" % (item, self.__class__.__name__))
 
@@ -312,7 +312,7 @@
 ######################################################################
 
 CONFLICTING = [BTree.insert, BTree.remove]
-OPS = [BTree.__contains__] * 98 + CONFLICTING
+OPS = [BTree.__contains__] * 198 + CONFLICTING
 
 ITEM_RANGE = 10000
 
@@ -341,20 +341,23 @@
 
 
 
-def run(threads=2, operations=2000000):
+def run(threads=2, concurrency=8, operations=2000000):
     threads = int(threads)
     operations = int(operations)
 
-    set_thread_pool(ThreadPool(threads))
+    tp = ThreadPool(threads)
+    print "new tp"
+    set_thread_pool(tp)
+    print "tp active"
 
     tree = BTree(20)
     for _ in xrange(1000):
         tree.insert(random.randint(1, ITEM_RANGE))
 
-    c_len = operations // threads
+    c_len = operations // concurrency
     fs = []
     parallel_time = time.time()
-    for i in xrange(threads):
+    for i in xrange(concurrency):
         fs.append(Future(task, i, tree, c_len))
     for f in fs:
         f()
@@ -369,6 +372,28 @@
 
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    cs, ops = int(argv[2]), int(argv[3])
+
+    print "params (iters, threads, concs, ops):", warmiters, threads, cs, ops
+
+    print "do warmup:"
+    for i in range(3):
+        print "iter", i, "time:", run(threads, cs, ops)
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        times.append(run(threads, cs, ops))
+    print "warmiters:", times
 
 if __name__ == '__main__':
-    run()
+    import sys
+    main(sys.argv[1:])
diff --git a/multithread/config-btree.json b/multithread/config-btree.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-btree.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 3,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "btree": {
+            "file": "btree/btree.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 3,
+            "warmiters": 3,
+            "args": ["16", "500000"]
+        }
+    }
+}
diff --git a/multithread/config-mandelbrot.json b/multithread/config-mandelbrot.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-mandelbrot.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "mandelbrot": {
+            "file": "mandelbrot/mandelbrot.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 3,
+            "warmiters": 3,
+            "args": ["64", "512", "512"]
+        }
+    }
+}
diff --git a/multithread/config-nqueens.json b/multithread/config-nqueens.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-nqueens.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "nqueens": {
+            "file": "nqueens/nqueens.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 3,
+            "warmiters": 3,
+            "args": ["10"]
+        }
+    }
+}
diff --git a/multithread/config-richards.json b/multithread/config-richards.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-richards.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "richards": {
+            "file": "richards/richards.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 3,
+            "warmiters": 3,
+            "args": ["30"]
+        }
+    }
+}
diff --git a/multithread/config-skiplist.json b/multithread/config-skiplist.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-skiplist.json
@@ -0,0 +1,21 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 3,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "skiplist": {
+            "file": "skiplist/skiplist.py",
+            "PYTHONPATH": "..",
+            "vmstarts": 3,
+            "warmiters": 3,
+            "args": ["16", "200000"]
+        }
+    }
+}
diff --git a/multithread/mandelbrot/mandelbrot.py b/multithread/mandelbrot/mandelbrot.py
--- a/multithread/mandelbrot/mandelbrot.py
+++ b/multithread/mandelbrot/mandelbrot.py
@@ -1,15 +1,17 @@
-from common.abstract_threading import atomic, Future, set_thread_pool, ThreadPool
+from common.abstract_threading import (
+    atomic, Future, set_thread_pool, ThreadPool,
+    turn_jitting_off)
 import sys, time
 
 
 def calculate(a, b, im_size, max_iter=255):
-    print "a:%s, b:%s, im_size:%s" % (a, b, im_size)
+    #print "a:%s, b:%s, im_size:%s" % (a, b, im_size)
     ar, ai = a
     br, bi = b
     width, height = im_size
     imag_step = (bi - ai) / (height - 1)
     real_step = (br - ar) / (width - 1)
-    print "real/width:%s, imag/height:%s" % (real_step, imag_step)
+    #print "real/width:%s, imag/height:%s" % (real_step, imag_step)
 
     result = [[0] * width for y in xrange(height)]
     for y in xrange(height):
@@ -82,8 +84,32 @@
     return parallel_time
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    stripes, w, h = int(argv[2]), int(argv[3]), int(argv[4])
+
+    print "params (iters, threads, s, w, h):", warmiters, threads, stripes, w, h
+
+    print "do warmup:"
+    for i in range(3):
+        print "iter", i, "time:", run(threads, stripes, w, h)
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        times.append(run(threads, stripes, w, h))
+    print "warmiters:", times
 
 if __name__ == '__main__':
-    image = run(int(sys.argv[1]), int(sys.argv[2]))
-    #save_to_file(out_image)
-    save_img(out_image)
+    main(sys.argv[1:])
+
+# if __name__ == '__main__':
+#     image = run(int(sys.argv[1]), int(sys.argv[2]))
+#     #save_to_file(out_image)
+#     save_img(out_image)
diff --git a/multithread/nqueens/nqueens.py b/multithread/nqueens/nqueens.py
--- a/multithread/nqueens/nqueens.py
+++ b/multithread/nqueens/nqueens.py
@@ -8,10 +8,10 @@
 
 
 import sys
-import time, random
+import time
 from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
-    hint_commit_soon)
+    hint_commit_soon, turn_jitting_off)
 
 from itertools import permutations
 import itertools
@@ -28,7 +28,7 @@
 
 def check_solutions(n, cols, perms):
     sols = []
-    with atomic:
+    if 1: #with atomic:
         for vec in perms:
             if n == len(set(vec[i]+i for i in cols)) \
                == len(set(vec[i]-i for i in cols)):
@@ -41,7 +41,6 @@
     fs = []
     cols = range(n)
     for perms in chunks(permutations(cols), 100000):
-        hint_commit_soon()
         fs.append(Future(check_solutions, n, cols, perms))
     print "Futures:", len(fs)
     for f in fs:
@@ -58,11 +57,36 @@
 
     find_solutions(n)
 
-
     # shutdown current pool
     set_thread_pool(None)
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    n = int(argv[2])
+
+    print "params (iters, threads, n):", warmiters, threads, n
+
+    print "do warmup:"
+    for i in range(3):
+        t = time.time()
+        run(threads, n)
+        print "iter", i, "time:", time.time() - t
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        t = time.time()
+        run(threads, n)
+        times.append(time.time() - t)
+    print "warmiters:", times
 
 if __name__ == "__main__":
-    run()
+    import sys
+    main(sys.argv[1:])
diff --git a/multithread/richards/richards.py b/multithread/richards/richards.py
new file mode 100755
--- /dev/null
+++ b/multithread/richards/richards.py
@@ -0,0 +1,471 @@
+# based on a Java version:
+#  Based on original version written in BCPL by Dr Martin Richards
+#  in 1981 at Cambridge University Computer Laboratory, England
+#  and a C++ version derived from a Smalltalk version written by
+#  L Peter Deutsch.
+#  Java version:  Copyright (C) 1995 Sun Microsystems, Inc.
+#  Translation from C++, Mario Wolczko
+#  Outer loop added by Alex Jacoby
+
+import thread, os
+from common.abstract_threading import atomic, hint_commit_soon
+
+# Task IDs
+I_IDLE = 1
+I_WORK = 2
+I_HANDLERA = 3
+I_HANDLERB = 4
+I_DEVA = 5
+I_DEVB = 6
+
+# Packet types
+K_DEV = 1000
+K_WORK = 1001
+
+# Packet
+
+BUFSIZE = 4
+
+BUFSIZE_RANGE = range(BUFSIZE)
+
+class Packet(object):
+    def __init__(self,l,i,k):
+        self.link = l
+        self.ident = i
+        self.kind = k
+        self.datum = 0
+        self.data = [0] * BUFSIZE
+
+    def append_to(self,lst):
+        self.link = None
+        if lst is None:
+            return self
+        else:
+            p = lst
+            next = p.link
+            while next is not None:
+                p = next
+                next = p.link
+            p.link = self
+            return lst
+
+# Task Records
+
+class TaskRec(object):
+    pass
+
+class DeviceTaskRec(TaskRec):
+    def __init__(self):
+        self.pending = None
+
+class IdleTaskRec(TaskRec):
+    def __init__(self):
+        self.control = 1
+        self.count = 10000
+
+class HandlerTaskRec(TaskRec):
+    def __init__(self):
+        self.work_in = None
+        self.device_in = None
+
+    def workInAdd(self,p):
+        self.work_in = p.append_to(self.work_in)
+        return self.work_in
+
+    def deviceInAdd(self,p):
+        self.device_in = p.append_to(self.device_in)
+        return self.device_in
+
+class WorkerTaskRec(TaskRec):
+    def __init__(self):
+        self.destination = I_HANDLERA
+        self.count = 0
+# Task
+
+class TaskState(object):
+    def __init__(self):
+        self.packet_pending = True
+        self.task_waiting = False
+        self.task_holding = False
+
+    def packetPending(self):
+        self.packet_pending = True
+        self.task_waiting = False
+        self.task_holding = False
+        return self
+
+    def waiting(self):
+        self.packet_pending = False
+        self.task_waiting = True
+        self.task_holding = False
+        return self
+
+    def running(self):
+        self.packet_pending = False
+        self.task_waiting = False
+        self.task_holding = False
+        return self
+
+    def waitingWithPacket(self):
+        self.packet_pending = True
+        self.task_waiting = True
+        self.task_holding = False
+        return self
+
+    def isPacketPending(self):
+        return self.packet_pending
+
+    def isTaskWaiting(self):
+        return self.task_waiting
+
+    def isTaskHolding(self):
+        return self.task_holding
+
+    def isTaskHoldingOrWaiting(self):
+        return self.task_holding or (not self.packet_pending and self.task_waiting)
+
+    def isWaitingWithPacket(self):
+        return self.packet_pending and self.task_waiting and not self.task_holding
+
+
+
+
+
+tracing = False
+layout = 0
+
+def trace(a):
+    global layout
+    layout -= 1
+    if layout <= 0:
+        print
+        layout = 50
+    print a,
+
+
+TASKTABSIZE = 10
+
+class TaskWorkArea(object):
+    def __init__(self):
+        self.taskTab = [None] * TASKTABSIZE
+
+        self.taskList = None
+
+        self.holdCount = 0
+        self.qpktCount = 0
+
+class Task(TaskState):
+
+
+    def __init__(self,i,p,w,initialState,r, taskWorkArea):
+        self.taskWorkArea = taskWorkArea
+        self.link = taskWorkArea.taskList
+        self.ident = i
+        self.priority = p
+        self.input = w
+
+        self.packet_pending = initialState.isPacketPending()
+        self.task_waiting = initialState.isTaskWaiting()
+        self.task_holding = initialState.isTaskHolding()
+
+        self.handle = r
+
+        taskWorkArea.taskList = self
+        taskWorkArea.taskTab[i] = self
+
+    def fn(self,pkt,r):
+        raise NotImplementedError
+
+
+    def addPacket(self,p,old):
+        if self.input is None:
+            self.input = p
+            self.packet_pending = True
+            if self.priority > old.priority:
+                return self
+        else:
+            p.append_to(self.input)
+        return old
+
+
+    def runTask(self):
+        if self.isWaitingWithPacket():
+            msg = self.input
+            self.input = msg.link
+            if self.input is None:
+                self.running()
+            else:
+                self.packetPending()
+        else:
+            msg = None
+
+        return self.fn(msg,self.handle)
+
+
+    def waitTask(self):
+        self.task_waiting = True
+        return self
+
+
+    def hold(self):
+        self.taskWorkArea.holdCount += 1
+        self.task_holding = True
+        return self.link
+
+
+    def release(self,i):
+        t = self.findtcb(i)
+        t.task_holding = False
+        if t.priority > self.priority:
+            return t
+        else:
+            return self
+
+
+    def qpkt(self,pkt):
+        t = self.findtcb(pkt.ident)
+        self.taskWorkArea.qpktCount += 1
+        pkt.link = None
+        pkt.ident = self.ident
+        return t.addPacket(pkt,self)
+
+
+    def findtcb(self,id):
+        t = self.taskWorkArea.taskTab[id]
+        if t is None:
+            raise Exception("Bad task id %d" % id)
+        return t
+
+
+# DeviceTask
+
+
+class DeviceTask(Task):
+    def __init__(self,i,p,w,s,r, taskWorkArea):
+        Task.__init__(self,i,p,w,s,r, taskWorkArea)
+
+    def fn(self,pkt,r):
+        d = r
+        assert isinstance(d, DeviceTaskRec)
+        if pkt is None:
+            pkt = d.pending
+            if pkt is None:
+                return self.waitTask()
+            else:
+                d.pending = None
+                return self.qpkt(pkt)
+        else:
+            d.pending = pkt
+            if tracing: trace(pkt.datum)
+            return self.hold()
+
+
+
+class HandlerTask(Task):
+    def __init__(self,i,p,w,s,r, taskWorkArea):
+        Task.__init__(self,i,p,w,s,r, taskWorkArea)
+
+    def fn(self,pkt,r):
+        h = r
+        assert isinstance(h, HandlerTaskRec)
+        if pkt is not None:
+            if pkt.kind == K_WORK:
+                h.workInAdd(pkt)
+            else:
+                h.deviceInAdd(pkt)
+        work = h.work_in
+        if work is None:
+            return self.waitTask()
+        count = work.datum
+        if count >= BUFSIZE:
+            h.work_in = work.link
+            return self.qpkt(work)
+
+        dev = h.device_in
+        if dev is None:
+            return self.waitTask()
+
+        h.device_in = dev.link
+        dev.datum = work.data[count]
+        work.datum = count + 1
+        return self.qpkt(dev)
+
+# IdleTask
+
+
+class IdleTask(Task):
+    def __init__(self,i,p,w,s,r, taskWorkArea):
+        Task.__init__(self,i,0,None,s,r, taskWorkArea)
+
+    def fn(self,pkt,r):
+        i = r
+        assert isinstance(i, IdleTaskRec)
+        i.count -= 1
+        if i.count == 0:
+            return self.hold()
+        elif i.control & 1 == 0:
+            i.control /= 2
+            return self.release(I_DEVA)
+        else:
+            i.control = i.control/2 ^ 0xd008
+            return self.release(I_DEVB)
+
+
+# WorkTask
+
+
+A = ord('A')
+
+class WorkTask(Task):
+    def __init__(self,i,p,w,s,r, taskWorkArea):
+        Task.__init__(self,i,p,w,s,r, taskWorkArea)
+
+    def fn(self,pkt,r):
+        w = r
+        assert isinstance(w, WorkerTaskRec)
+        if pkt is None:
+            return self.waitTask()
+
+        if w.destination == I_HANDLERA:
+            dest = I_HANDLERB
+        else:
+            dest = I_HANDLERA
+
+        w.destination = dest
+        pkt.ident = dest
+        pkt.datum = 0
+
+        for i in BUFSIZE_RANGE: # xrange(BUFSIZE)
+            w.count += 1
+            if w.count > 26:
+                w.count = 1
+            pkt.data[i] = A + w.count - 1
+
+        return self.qpkt(pkt)
+
+try:
+    from time import time
+except ImportError:
+    def time():
+        return 0
+
+
+def schedule(taskWorkArea):
+    t = taskWorkArea.taskList
+    while t is not None:
+        pkt = None
+
+        if tracing:
+            print "tcb =",t.ident
+
+        if t.isTaskHoldingOrWaiting():
+            t = t.link
+        else:
+            if tracing: trace(chr(ord("0")+t.ident))
+            t = t.runTask()
+
+class Richards(object):
+
+    def __init__(self):
+        self.finished_lock = thread.allocate_lock()
+        self.finished_lock.acquire()
+
+    def run_and_unlock(self, count):
+        print 'running...'
+        iterations = 0
+        self.result = True
+        for i in range(count):
+            self.result = self.run()
+        print 'done, iterations=%d, result=%r' % (count, self.result)
+        self.finished_lock.release()
+
+    def run(self):
+        #hint_commit_soon()
+        #with atomic:
+        if 1:
+            taskWorkArea = TaskWorkArea()
+
+            IdleTask(I_IDLE, 1, 10000, TaskState().running(), IdleTaskRec(),
+                     taskWorkArea)
+
+            wkq = Packet(None, 0, K_WORK)
+            wkq = Packet(wkq , 0, K_WORK)
+            WorkTask(I_WORK, 1000, wkq, TaskState().waitingWithPacket(), WorkerTaskRec(),
+                     taskWorkArea)
+
+            wkq = Packet(None, I_DEVA, K_DEV)
+            wkq = Packet(wkq , I_DEVA, K_DEV)
+            wkq = Packet(wkq , I_DEVA, K_DEV)
+            HandlerTask(I_HANDLERA, 2000, wkq, TaskState().waitingWithPacket(), HandlerTaskRec(),
+                        taskWorkArea)
+
+            wkq = Packet(None, I_DEVB, K_DEV)
+            wkq = Packet(wkq , I_DEVB, K_DEV)
+            wkq = Packet(wkq , I_DEVB, K_DEV)
+            HandlerTask(I_HANDLERB, 3000, wkq, TaskState().waitingWithPacket(), HandlerTaskRec(),
+                        taskWorkArea)
+
+            wkq = None;
+            DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec(),
+                       taskWorkArea)
+            DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec(),
+                       taskWorkArea)
+
+            schedule(taskWorkArea)
+
+            if taskWorkArea.holdCount == 9297 and taskWorkArea.qpktCount == 23246:
+                pass
+            else:
+                return False
+        #hint_commit_soon()
+
+        return True
+
+def entry_point(iterations, NUM_THREADS):
+    rlist = [Richards() for i in range(NUM_THREADS)]
+    startTime = time()
+    for i, r in enumerate(rlist):
+        count = (iterations * (i + 1)) // NUM_THREADS
+        count -= (iterations * i) // NUM_THREADS
+        thread.start_new_thread(r.run_and_unlock, (count,))
+    for r in rlist:
+        r.finished_lock.acquire()
+    endTime = time()
+    result = all(r.result for r in rlist)
+    return result, startTime, endTime
+
+def main(argv):
+    #def main(entry_point = entry_point, iterations = 10, threads = 4):
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    iterations = int(argv[2])
+
+    print "params:", warmiters, threads, iterations
+    print "do warmup:"
+    result, startTime, endTime = entry_point(4, threads)
+
+    print "do", warmiters, "real iters:"
+    times = []
+    import gc
+    for i in range(warmiters):
+        gc.collect()
+
+        print "Richards benchmark (Python) starting..."
+        result, startTime, endTime = entry_point(iterations, threads)
+        if not result:
+            print "Incorrect results!"
+            return -1
+        print "finished."
+        total_s = endTime - startTime
+        print "Total time for %d iterations: %.2f secs" %(iterations,total_s)
+        print "Average time per iteration: %.2f ms" %(total_s*1000/iterations)
+
+        times.append(total_s)
+    print "warmiters:", times
+
+
+    return 0
+
+if __name__ == '__main__':
+    import sys
+    main(sys.argv[1:])
diff --git a/multithread/skiplist/skiplist.py b/multithread/skiplist/skiplist.py
--- a/multithread/skiplist/skiplist.py
+++ b/multithread/skiplist/skiplist.py
@@ -1,8 +1,9 @@
+
 # https://github.com/kunigami/blog-examples/tree/master/2012-09-23-skip-list
 
 from common.abstract_threading import (atomic, Future,
                                        set_thread_pool, ThreadPool,
-                                       print_abort_info, hint_commit_soon)
+                                       hint_commit_soon)
 import time, threading
 
 import random
@@ -66,7 +67,7 @@
                 node.next[i] = update[i].next[i]
                 update[i].next[i] = node
             self.len += 1
-        hint_commit_soon()
+        #hint_commit_soon()
 
     def remove(self, elem):
         update = self.updateList(elem)
@@ -78,7 +79,7 @@
                 if self.head.next[i] == None:
                     self.maxHeight -= 1
             self.len -= 1
-            hint_commit_soon()
+            #hint_commit_soon()
 
     def printList(self):
         for i in range(len(self.head.next)-1, -1, -1):
@@ -90,7 +91,7 @@
 
 
 CONFLICTING = [SkipList.insert, SkipList.remove]
-OPS = [SkipList.find] * 98 + CONFLICTING
+OPS = [SkipList.find] * 198 + CONFLICTING
 ITEM_RANGE = 1000000
 
 def task(id, slist, ops):
@@ -119,7 +120,7 @@
 
 
 
-def run(threads=2, operations=2000000):
+def run(threads=2, concurrency=8, operations=2000000):
     threads = int(threads)
     operations = int(operations)
 
@@ -130,10 +131,10 @@
     for _ in xrange(1000):
         slist.insert(random.randint(1, ITEM_RANGE))
 
-    c_len = operations // threads
+    c_len = operations // concurrency
     fs = []
     parallel_time = time.time()
-    for i in xrange(threads):
+    for i in xrange(concurrency):
         fs.append(Future(task, i, slist, c_len))
     for f in fs:
         f()
@@ -149,7 +150,28 @@
 
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    cs, ops = int(argv[2]), int(argv[3])
 
+    print "params (iters, threads, concs, ops):", warmiters, threads, cs, ops
+
+    print "do warmup:"
+    for i in range(3):
+        print "iter", i, "time:", run(threads, cs, ops)
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        times.append(run(threads, cs, ops))
+    print "warmiters:", times
 
 if __name__ == '__main__':
-    run()
+    import sys
+    main(sys.argv[1:])

From pypy.commits at gmail.com  Mon Jun 20 07:04:41 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:41 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add mersenne config
Message-ID: <5767cdc9.89acc20a.65989.ffffd7bb@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r356:0dbc353ce596
Date: 2016-05-27 16:10 +0200
http://bitbucket.org/pypy/benchmarks/changeset/0dbc353ce596/

Log:	add mersenne config

diff --git a/multithread/config-mersenne.json b/multithread/config-mersenne.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-mersenne.json
@@ -0,0 +1,19 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 3,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "mersenne": {
+            "file": "mersenne/mersenne.py",
+            "PYTHONPATH": "..",
+            "args": ["1500"]
+        }
+    }
+}
diff --git a/multithread/config-richards.json b/multithread/config-richards.json
--- a/multithread/config-richards.json
+++ b/multithread/config-richards.json
@@ -3,7 +3,7 @@
         "file": null,
         "threads": [1, 2, 4, 8],
         "vmstarts": 3,
-        "warmiters": 5,
+        "warmiters": 3,
         "PYTHONPATH": ".",
         "args": [],
         "cwd": "."
@@ -13,8 +13,6 @@
         "richards": {
             "file": "richards/richards.py",
             "PYTHONPATH": "..",
-            "vmstarts": 3,
-            "warmiters": 3,
             "args": ["30"]
         }
     }
diff --git a/multithread/mersenne/mersenne.py b/multithread/mersenne/mersenne.py
--- a/multithread/mersenne/mersenne.py
+++ b/multithread/mersenne/mersenne.py
@@ -6,7 +6,7 @@
 import time, random
 from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
-    hint_commit_soon, print_abort_info)
+    turn_jitting_off)
 
 import itertools
 from collections import deque
@@ -48,10 +48,11 @@
         return
 
     for p in ps:
-        with atomic:
+        if 1: #with atomic:
             if is_prime(p) and is_mersenne_prime(p):
                 #print p
-                counter[0] += 1
+                with atomic:
+                    counter[0] += 1
         if counter[0] >= upb_count:
             break
 
@@ -84,6 +85,32 @@
     return
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    n = int(argv[2])
 
-if __name__ == "__main__":
-    run()
+    print "params (iters, threads, n):", warmiters, threads, n
+
+    print "do warmup:"
+    for i in range(2):
+        t = time.time()
+        run(threads, n)
+        print "iter", i, "time:", time.time() - t
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        t = time.time()
+        run(threads, n)
+        times.append(time.time() - t)
+    print "warmiters:", times
+
+if __name__ == '__main__':
+    import sys
+    main(sys.argv[1:])
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -156,7 +156,6 @@
                 indent=4, separators=(',', ': ')))
 
 
-if __name__ != '__main__': #FIXME: emacs bug?
+if __name__ == '__main__':
     main(sys.argv[1:])
-else:
-    main(["pypy-c", "config-raytrace.json", "results.json",])
+    # main(["pypy-c", "config-mersenne.json", "results.json",])

From pypy.commits at gmail.com  Mon Jun 20 07:04:46 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:46 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add initial short,
 testing config
Message-ID: <5767cdce.0410c20a.4e6ac.0a1a@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r358:6a4ff9f48f62
Date: 2016-05-28 09:05 +0200
http://bitbucket.org/pypy/benchmarks/changeset/6a4ff9f48f62/

Log:	add initial short, testing config

diff --git a/multithread/config-all-short.json b/multithread/config-all-short.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-all-short.json
@@ -0,0 +1,62 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 1,
+        "warmiters": 1,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "nqueens": {
+            "file": "nqueens/nqueens.py",
+            "PYTHONPATH": "..",
+            "args": ["10"]
+        },
+
+        "parsible-bench": {
+            "file": "parsible-bench/parsible-bench.py",
+            "PYTHONPATH": ".."
+        },
+
+        "mersenne": {
+            "file": "mersenne/mersenne.py",
+            "PYTHONPATH": "..",
+            "args": ["1500"]
+        },
+
+        "richards": {
+            "file": "richards/richards.py",
+            "PYTHONPATH": "..",
+            "args": ["20"]
+        },
+
+        "mandelbrot": {
+            "file": "mandelbrot/mandelbrot.py",
+            "PYTHONPATH": "..",
+            "args": ["64", "128", "512"]
+        },
+
+        "btree": {
+            "file": "btree/btree.py",
+            "PYTHONPATH": "..",
+            "args": ["16", "300000"]
+        },
+
+        "skiplist": {
+            "file": "skiplist/skiplist.py",
+            "PYTHONPATH": "..",
+            "args": ["16", "100000"]
+        },
+
+        "raytrace": {
+            "file": "raytrace/raytrace.py",
+            "PYTHONPATH": "..",
+            "args": ["128", "512"]
+        }
+
+    }
+
+}

From pypy.commits at gmail.com  Mon Jun 20 07:04:44 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:44 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add parsible benchmark
Message-ID: <5767cdcc.0ed11c0a.a826e.ffffa28f@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r357:53cf5fa8e7aa
Date: 2016-05-27 17:49 +0200
http://bitbucket.org/pypy/benchmarks/changeset/53cf5fa8e7aa/

Log:	add parsible benchmark

diff too long, truncating to 2000 out of 100736 lines

diff --git a/multithread/btree/btree.py b/multithread/btree/btree.py
--- a/multithread/btree/btree.py
+++ b/multithread/btree/btree.py
@@ -345,10 +345,7 @@
     threads = int(threads)
     operations = int(operations)
 
-    tp = ThreadPool(threads)
-    print "new tp"
-    set_thread_pool(tp)
-    print "tp active"
+    set_thread_pool(ThreadPool(threads))
 
     tree = BTree(20)
     for _ in xrange(1000):
diff --git a/multithread/config-parsible.json b/multithread/config-parsible.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-parsible.json
@@ -0,0 +1,18 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "parsible-bench": {
+            "file": "parsible-bench/parsible-bench.py",
+            "PYTHONPATH": ".."
+        }
+    }
+}
diff --git a/multithread/parsible-bench/CHANGELOG.md b/multithread/parsible-bench/CHANGELOG.md
new file mode 100644
--- /dev/null
+++ b/multithread/parsible-bench/CHANGELOG.md
@@ -0,0 +1,7 @@
+# February 26th, 2014
+
+* Add `proftpd` log file parser. Thanks to @libertyy
+
+# February 10th, 2014
+
+* Updated regex for parsing Nginx `timed-combined` format to allow for 1 or 2 spaces between the timestamp and message. Thanks to @wflynnvey.
\ No newline at end of file
diff --git a/multithread/parsible-bench/LICENSE.md b/multithread/parsible-bench/LICENSE.md
new file mode 100644
--- /dev/null
+++ b/multithread/parsible-bench/LICENSE.md
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Andrew Gross
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/multithread/parsible-bench/README.md b/multithread/parsible-bench/README.md
new file mode 100644
--- /dev/null
+++ b/multithread/parsible-bench/README.md
@@ -0,0 +1,142 @@
+# Parsible
+
+A tool to help you parse your log files, written in [Python](http://http://python.org/).  The goal was to make a tool that will do the grunt work of following your logs in realtime, and to be easily be extended via plugins. Check out our tech blog post on why we wanted Parsible [here]()http://tech.yipit.com/2012/08/03/parsible-straightforward-log-parsing/.
+
+
+## Concepts
+===========
+
+There are a few core ideas we tried to bring together in Parsible:
+
+* **Plugins**:  We wanted to make the parsers, processors, and outputs all customizable and easy to write. All plugins are autodiscovered on startup and have a very simple format for connecting them together.
+
+* **Real Time**: Parsible will tail your log file as the lines come in, we opted away from a stateful approach where new lines are read in batches since we feel it simplifies the flow and reduces complexity.
+
+* **Batch Processing**: Parsible has a switch to modify the behavior so that it acts like a standard parser.  Instead of tailing from the end of the file, it will start at the beginning and exit once it has reached the last line.
+
+* **Generators**:  Log files can get big, really big.  By leveraging generators Parsible can keep it's memory footprint small and independent of the size of log file. There is no hard restriction on memory, disk, or CPU usage, so be careful when writing your custom plugins.
+
+* **System Conventions**: Since Parsible works with logs it is wise to follow Linux logging conventions.  Parsible integrates easily with [logrotate](http://linuxcommand.org/man_pages/logrotate8.html).
+
+## Plugins
+===========
+
+**Parsers**: We wanted to make it easy to write custom parsers for whatever type of log file you may be reading.  Your custom parser has to follow a few conventions.  A simple nginx parser has been included as an example.
+
+1. Your parser should live inside of the `plugins/parsers` directory
+
+2. Your parsing function should start with `parse_`, take for example the included nginx parser which contains a function called `parse_nginx`
+
+3. The parsing method signature should take one parameter which will consist of one line from the log file.  You may parse the line however you see fit, we opted for a regex implementation since it fits nicely with our dictionary output format and we expect our nginx log data to be well structured.
+
+4. The parsing method can output whatever you like, as it will be fed directly into the processing functions.  In our case we found that a dictionary works very well as lightweight storage for the parsed data although this is not required as you get to write the processing functions as well.
+
+5. Errors from a `parse` method are swallowed by the same try/except block that handles `process` methods due to lazy evaluation.  Currently there is no recording of these occurrences although this behavior can be easily modified.
+
+***
+
+**Processors**: Once a line of log data is parsed it is time to do something useful with it.  You can have your processors do whatever you wish, although it is suggested that they remain stateless so that you don't have any lingering effects from feeding on large log files over the course of a day. Some sample `process` methods can be found in `plugins/outputs/url.py.`
+
+1. Your processors should live inside of the `plugins/processors` directory
+
+2. Your parsing function should start with `process_` so that the autoloader can find it. For example the sample processor contains  functions called `process_api` and `process_business`.
+
+3. Your parsing method can take in whatever you please.  The output of your parsing function will be fed directly to each processor that Parsible was able to discover.
+
+4. Outputting from parsing methods is up to you.  The suggested flow is that you import any output functions you wish to use directly and call them as needed.
+
+5. Any errors from a `process` method are currently swallowed and left untracked, although it is very simple to modify this behavior if desired.
+
+***
+
+**Outputs**: Output functions are given their own directory to simplify the structure of Parsible.  The output functions should be called directly by your code in your `process` methods, but it is cleaner to logically separate them inside the plugin system for clarity.  Parsible will not attempt to run any `output` functions directly.  For some example `output` functions check out `plugins/outputs/statsd.py`
+
+***
+
+## System Conventions
+======================
+
+**Log Rotate**: We like our code to play nice with our systems, especially for a program like Parsible.
+
+* Parsible creates a PID file in `/tmp/parsible.pid` so that it is easy to find with other programs.
+
+* Parsible reloads the log file on receipt of the USR1 signal. This makes it easy to work with inside of [logrotate](http://linuxcommand.org/man_pages/logrotate8.html) scripts.
+
+Here is our `logrotate` script before Parsible:
+
+```bash
+    postrotate
+        [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
+    endscript
+```
+
+And After
+
+```bash
+    postrotate
+        [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`; \
+        [ ! -f /tmp/parsible.pid ] || kill -USR1 `cat /tmp/parsible.pid`
+    endscript
+```
+
+If you don't care to set up `logrotate` or `logrotate` does not apply, just use `--auto-reload True` and it will try to reload the log file after 10 seconds of inactivity.
+
+## Usage
+=========
+
+1. Clone Parsible
+2. Write your parser (or use one someone else wrote!)
+3. Figure out how you want to process your log lines and write some processors
+4. Set up any outputs you want to use
+5. Run it! (We keep ours running under `supervisord`, although we have not had issues with crashes.)
+
+
+```bash
+parsible.py --log-file /var/log/mylog --pid-file /tmp/parsible.pid --parser parse_nginx
+```
+
+To add debug messages regarding errors that my have been swallowed by Parsible add the `--debug True` option to your command line arguments.  This can be relatively verbose since it can create multiple messages per processed line so it is not the recommended production configuration.
+
+To enable batch processing mode, just append `--batch-mode True` to your command line invocation and Parsible will act as a standard parser that exits at the end of the file.  This can be useful for backfilling data or doing ad hoc analysis of old files.
+
+## Requirements
+================
+
+* Linux
+* Python 2.7+ (due to argparse)
+* Some tasty logs
+
+
+## Warnings
+============
+
+Parsible does not gaurantee that every line of your log file will get parsed.  When it is first started Parsible seeks to the end of the log file. Additionally, whenever the `USR1` signal is received Parsible will attempt to load the file at the configured location.  There is no logic to make sure the current file is fully parsed before switching. This can lead to some lines not being processed during the switchover.  If this is a major issue for you please feel free to submit a feature request.
+
+Although Parsible is designed to be lightweight it does not gaurantee it.  User created plugins have no restrictions on their behavior and can monopolize resources as they see fit.
+
+Parsible grabs a 'line' based on the return of file.readline().  This means it usually won't handle multiline exceptions very well.  Feel free to request the feature if you want it added.
+
+The mispelling of the name of the project is intentional (vs Parsable)
+
+##  Contribute
+==============
+
+If you are interested in contributing to Parsible here are the steps:
+
+1. fork Parsible from here: http://github.com/Yipit/parsible
+2. Clone your fork
+3. Hack away
+4. If you are adding new functionality, document it in the README
+5. If necessary, rebase your commits into logical chunks, without errors
+6. Push the branch up to GitHub
+7. Send a pull request to the Yipit/parsible project.
+8. We'll take a look and try to get your changes in!
+
+## Contributors
+================
+
+None for now, but feel free to check the [commit history](https://github.com/Yipit/parsible/commits/master)!
+
+A special thanks to the fine folks at [Etsy](http://www.etsy.com) for publishing the [StatsD](http://github.com/etsy/statsd) project which gave me an excellent README to use as a template.
+
+
diff --git a/multithread/parsible-bench/data/nasa-http-log b/multithread/parsible-bench/data/nasa-http-log
new file mode 100644
--- /dev/null
+++ b/multithread/parsible-bench/data/nasa-http-log
@@ -0,0 +1,100000 @@
+199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] "GET /history/apollo/ HTTP/1.0" 200 6245
+unicomp6.unicomp.net - - [01/Jul/1995:00:00:06 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+199.120.110.21 - - [01/Jul/1995:00:00:09 -0400] "GET /shuttle/missions/sts-73/mission-sts-73.html HTTP/1.0" 200 4085
+burger.letters.com - - [01/Jul/1995:00:00:11 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+199.120.110.21 - - [01/Jul/1995:00:00:11 -0400] "GET /shuttle/missions/sts-73/sts-73-patch-small.gif HTTP/1.0" 200 4179
+burger.letters.com - - [01/Jul/1995:00:00:12 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+burger.letters.com - - [01/Jul/1995:00:00:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 0
+205.212.115.106 - - [01/Jul/1995:00:00:12 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+d104.aa.net - - [01/Jul/1995:00:00:13 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+129.94.144.152 - - [01/Jul/1995:00:00:13 -0400] "GET / HTTP/1.0" 200 7074
+unicomp6.unicomp.net - - [01/Jul/1995:00:00:14 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+unicomp6.unicomp.net - - [01/Jul/1995:00:00:14 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+unicomp6.unicomp.net - - [01/Jul/1995:00:00:14 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+d104.aa.net - - [01/Jul/1995:00:00:15 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+d104.aa.net - - [01/Jul/1995:00:00:15 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+d104.aa.net - - [01/Jul/1995:00:00:15 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+129.94.144.152 - - [01/Jul/1995:00:00:17 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 304 0
+199.120.110.21 - - [01/Jul/1995:00:00:17 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ppptky391.asahi-net.or.jp - - [01/Jul/1995:00:00:18 -0400] "GET /facts/about_ksc.html HTTP/1.0" 200 3977
+net-1-141.eden.com - - [01/Jul/1995:00:00:19 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.jpg HTTP/1.0" 200 34029
+ppptky391.asahi-net.or.jp - - [01/Jul/1995:00:00:19 -0400] "GET /images/launchpalms-small.gif HTTP/1.0" 200 11473
+205.189.154.54 - - [01/Jul/1995:00:00:24 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+waters-gw.starway.net.au - - [01/Jul/1995:00:00:25 -0400] "GET /shuttle/missions/51-l/mission-51-l.html HTTP/1.0" 200 6723
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:27 -0400] "GET / HTTP/1.0" 200 7074
+205.189.154.54 - - [01/Jul/1995:00:00:29 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+alyssa.prodigy.com - - [01/Jul/1995:00:00:33 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:35 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+dial22.lloyd.com - - [01/Jul/1995:00:00:37 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0613.jpg HTTP/1.0" 200 61716
+smyth-pc.moorecap.com - - [01/Jul/1995:00:00:38 -0400] "GET /history/apollo/apollo-13/images/70HC314.GIF HTTP/1.0" 200 101267
+205.189.154.54 - - [01/Jul/1995:00:00:40 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-orl2-01.ix.netcom.com - - [01/Jul/1995:00:00:41 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:41 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:41 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+205.189.154.54 - - [01/Jul/1995:00:00:41 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:41 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ppp-mia-30.shadow.net - - [01/Jul/1995:00:00:43 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ix-orl2-01.ix.netcom.com - - [01/Jul/1995:00:00:44 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+gayle-gaston.tenet.edu - - [01/Jul/1995:00:00:50 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+piweba3y.prodigy.com - - [01/Jul/1995:00:00:54 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+scheyer.clark.net - - [01/Jul/1995:00:00:58 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock-2.mpg HTTP/1.0" 200 49152
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:00:59 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0882.jpg HTTP/1.0" 200 77163
+199.72.81.55 - - [01/Jul/1995:00:00:59 -0400] "GET /history/ HTTP/1.0" 200 1382
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:02 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:04 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:04 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:04 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+dd14-012.compuserve.com - - [01/Jul/1995:00:01:05 -0400] "GET /shuttle/technology/images/srb_16-small.gif HTTP/1.0" 200 42732
+205.189.154.54 - - [01/Jul/1995:00:01:06 -0400] "GET /cgi-bin/imagemap/countdown?99,176 HTTP/1.0" 302 110
+205.189.154.54 - - [01/Jul/1995:00:01:08 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+www-a1.proxy.aol.com - - [01/Jul/1995:00:01:09 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+dd15-062.compuserve.com - - [01/Jul/1995:00:01:12 -0400] "GET /news/sci.space.shuttle/archive/sci-space-shuttle-22-apr-1995-40.txt HTTP/1.0" 404 -
+205.212.115.106 - - [01/Jul/1995:00:01:13 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+piweba3y.prodigy.com - - [01/Jul/1995:00:01:14 -0400] "GET /shuttle/technology/images/srb_mod_compare_3-small.gif HTTP/1.0" 200 55666
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:01:14 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:17 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+ix-orl2-01.ix.netcom.com - - [01/Jul/1995:00:01:18 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+smyth-pc.moorecap.com - - [01/Jul/1995:00:01:19 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+205.189.154.54 - - [01/Jul/1995:00:01:19 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0423.txt HTTP/1.0" 200 1224
+www-b4.proxy.aol.com - - [01/Jul/1995:00:01:21 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 70712
+smyth-pc.moorecap.com - - [01/Jul/1995:00:01:24 -0400] "GET /history/apollo/apollo-spacecraft.txt HTTP/1.0" 200 2261
+slip1.yab.com - - [01/Jul/1995:00:01:26 -0400] "GET /shuttle/resources/orbiters/endeavour.html HTTP/1.0" 200 6168
+link097.txdirect.net - - [01/Jul/1995:00:01:26 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:27 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:27 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:01:27 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+link097.txdirect.net - - [01/Jul/1995:00:01:27 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+slip1.yab.com - - [01/Jul/1995:00:01:29 -0400] "GET /shuttle/resources/orbiters/endeavour.gif HTTP/1.0" 200 16991
+link097.txdirect.net - - [01/Jul/1995:00:01:31 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+link097.txdirect.net - - [01/Jul/1995:00:01:31 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:32 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+port26.annex2.nwlink.com - - [01/Jul/1995:00:01:32 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+onyx.southwind.net - - [01/Jul/1995:00:01:34 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+onyx.southwind.net - - [01/Jul/1995:00:01:35 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+onyx.southwind.net - - [01/Jul/1995:00:01:39 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+unicomp6.unicomp.net - - [01/Jul/1995:00:01:41 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+199.72.81.55 - - [01/Jul/1995:00:01:43 -0400] "GET / HTTP/1.0" 200 7074
+link097.txdirect.net - - [01/Jul/1995:00:01:44 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+link097.txdirect.net - - [01/Jul/1995:00:01:45 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+199.72.81.55 - - [01/Jul/1995:00:01:46 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+gater4.sematech.org - - [01/Jul/1995:00:01:46 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+link097.txdirect.net - - [01/Jul/1995:00:01:47 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:01:49 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+gater3.sematech.org - - [01/Jul/1995:00:01:50 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+199.72.81.55 - - [01/Jul/1995:00:01:50 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+199.72.81.55 - - [01/Jul/1995:00:01:51 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+gater4.sematech.org - - [01/Jul/1995:00:01:52 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+199.72.81.55 - - [01/Jul/1995:00:01:52 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:01:52 -0400] "GET /software/winvn/userguide/wvnguide.html HTTP/1.0" 200 5998
+gater3.sematech.org - - [01/Jul/1995:00:01:52 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:01:53 -0400] "GET /cgi-bin/imagemap/countdown?102,174 HTTP/1.0" 302 110
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:01:55 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+link097.txdirect.net - - [01/Jul/1995:00:01:55 -0400] "GET /shuttle/resources/orbiters/columbia.html HTTP/1.0" 200 6922
+dave.dev1.ihub.com - - [01/Jul/1995:00:01:55 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+link097.txdirect.net - - [01/Jul/1995:00:01:56 -0400] "GET /shuttle/resources/orbiters/columbia-logo.gif HTTP/1.0" 200 11417
+netport-27.iu.net - - [01/Jul/1995:00:01:57 -0400] "GET / HTTP/1.0" 200 7074
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:01:57 -0400] "GET /software/winvn/userguide/wvnguide.gif HTTP/1.0" 200 4151
+dave.dev1.ihub.com - - [01/Jul/1995:00:01:58 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dave.dev1.ihub.com - - [01/Jul/1995:00:01:58 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dave.dev1.ihub.com - - [01/Jul/1995:00:01:58 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+pm13.j51.com - - [01/Jul/1995:00:01:58 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+bart-slip1.llnl.gov - - [01/Jul/1995:00:01:59 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+link097.txdirect.net - - [01/Jul/1995:00:01:59 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+link097.txdirect.net - - [01/Jul/1995:00:01:59 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+netport-27.iu.net - - [01/Jul/1995:00:02:00 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 304 0
+netport-27.iu.net - - [01/Jul/1995:00:02:01 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+netport-27.iu.net - - [01/Jul/1995:00:02:01 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 304 0
+netport-27.iu.net - - [01/Jul/1995:00:02:01 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 304 0
+smyth-pc.moorecap.com - - [01/Jul/1995:00:02:02 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+netport-27.iu.net - - [01/Jul/1995:00:02:04 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:04 -0400] "GET /software/winvn/userguide/winvnsm.gif HTTP/1.0" 200 3293
+dd14-046.compuserve.com - - [01/Jul/1995:00:02:05 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+205.189.154.54 - - [01/Jul/1995:00:02:11 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0869.gif HTTP/1.0" 200 45798
+port2.electrotex.com - - [01/Jul/1995:00:02:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 70712
+199.33.32.50 - - [01/Jul/1995:00:02:12 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+scheyer.clark.net - - [01/Jul/1995:00:02:13 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-launch-3.mpg HTTP/1.0" 200 130724
+199.33.32.50 - - [01/Jul/1995:00:02:14 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dynip42.efn.org - - [01/Jul/1995:00:02:14 -0400] "GET /software HTTP/1.0" 302 -
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:02:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0868.gif HTTP/1.0" 200 47122
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:02:15 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+smyth-pc.moorecap.com - - [01/Jul/1995:00:02:15 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+dynip42.efn.org - - [01/Jul/1995:00:02:15 -0400] "GET /software/ HTTP/1.0" 200 689
+waters-gw.starway.net.au - - [01/Jul/1995:00:02:16 -0400] "GET /shuttle/missions/51-l/51-l-info.html HTTP/1.0" 200 1387
+199.33.32.50 - - [01/Jul/1995:00:02:16 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 62761
+lmsmith.tezcat.com - - [01/Jul/1995:00:02:16 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+dynip42.efn.org - - [01/Jul/1995:00:02:17 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+dynip42.efn.org - - [01/Jul/1995:00:02:17 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+unicomp6.unicomp.net - - [01/Jul/1995:00:02:17 -0400] "GET /facilities/lcc.html HTTP/1.0" 200 2489
+lmsmith.tezcat.com - - [01/Jul/1995:00:02:19 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+lmsmith.tezcat.com - - [01/Jul/1995:00:02:20 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+unicomp6.unicomp.net - - [01/Jul/1995:00:02:20 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+unicomp6.unicomp.net - - [01/Jul/1995:00:02:21 -0400] "GET /images/kscmap-tiny.gif HTTP/1.0" 200 2537
+lmsmith.tezcat.com - - [01/Jul/1995:00:02:21 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:02:25 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.txt HTTP/1.0" 200 1391
+link097.txdirect.net - - [01/Jul/1995:00:02:25 -0400] "GET /shuttle/missions/sts-65/sts-65-patch-small.gif HTTP/1.0" 200 11757
+gayle-gaston.tenet.edu - - [01/Jul/1995:00:02:25 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+pme607.onramp.awinc.com - - [01/Jul/1995:00:02:25 -0400] "GET /shuttle/missions/sts-71/sts-71-day-04-highlights.html HTTP/1.0" 200 5544
+dynip42.efn.org - - [01/Jul/1995:00:02:26 -0400] "GET /software/winvn/ HTTP/1.0" 200 2244
+onyx.southwind.net - - [01/Jul/1995:00:02:27 -0400] "GET /cgi-bin/imagemap/countdown?103,146 HTTP/1.0" 302 96
+dynip42.efn.org - - [01/Jul/1995:00:02:28 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+dynip42.efn.org - - [01/Jul/1995:00:02:28 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+smyth-pc.moorecap.com - - [01/Jul/1995:00:02:30 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+usr7-dialup46.chicago.mci.net - - [01/Jul/1995:00:02:35 -0400] "GET /shuttle/missions/ HTTP/1.0" 200 12283
+smyth-pc.moorecap.com - - [01/Jul/1995:00:02:38 -0400] "GET /history/apollo/images/apollo-small.gif HTTP/1.0" 200 9630
+dynip42.efn.org - - [01/Jul/1995:00:02:39 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:40 -0400] "GET /software/winvn HTTP/1.0" 302 -
+gater3.sematech.org - - [01/Jul/1995:00:02:41 -0400] "GET /cgi-bin/imagemap/countdown?99,173 HTTP/1.0" 302 110
+dynip42.efn.org - - [01/Jul/1995:00:02:41 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+dynip42.efn.org - - [01/Jul/1995:00:02:41 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+dynip42.efn.org - - [01/Jul/1995:00:02:42 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:42 -0400] "GET /software/winvn/ HTTP/1.0" 200 2244
+gater4.sematech.org - - [01/Jul/1995:00:02:43 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:45 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:47 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:49 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:02:52 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:02:55 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0868.txt HTTP/1.0" 200 657
+waters-gw.starway.net.au - - [01/Jul/1995:00:02:57 -0400] "GET /shuttle/missions/51-l/images/ HTTP/1.0" 200 1038
+brandt.xensei.com - - [01/Jul/1995:00:02:57 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+sneaker.oregoncoast.com - - [01/Jul/1995:00:02:57 -0400] "GET /shuttle/countdown/ HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:02:58 -0400] "GET /msfc/astro_home.html HTTP/1.0" 304 0
+slip1.yab.com - - [01/Jul/1995:00:03:00 -0400] "GET /shuttle/missions/sts-49/mission-sts-49.html HTTP/1.0" 200 9379
+onyx.southwind.net - - [01/Jul/1995:00:03:00 -0400] "GET /cgi-bin/imagemap/countdown?102,210 HTTP/1.0" 302 95
+dd11-054.compuserve.com - - [01/Jul/1995:00:03:01 -0400] "GET / HTTP/1.0" 200 7074
+onyx.southwind.net - - [01/Jul/1995:00:03:02 -0400] "GET /shuttle/countdown/tour.html HTTP/1.0" 200 4347
+sneaker.oregoncoast.com - - [01/Jul/1995:00:03:02 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 304 0
+sneaker.oregoncoast.com - - [01/Jul/1995:00:03:02 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+sneaker.oregoncoast.com - - [01/Jul/1995:00:03:03 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+onyx.southwind.net - - [01/Jul/1995:00:03:03 -0400] "GET /images/KSC-94EC-412-small.gif HTTP/1.0" 200 20484
+gater3.sematech.org - - [01/Jul/1995:00:03:04 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0868.gif HTTP/1.0" 200 47122
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:03:05 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+brandt.xensei.com - - [01/Jul/1995:00:03:05 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+brandt.xensei.com - - [01/Jul/1995:00:03:08 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+piweba3y.prodigy.com - - [01/Jul/1995:00:03:09 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-tcdt-crew-walkout.mpg HTTP/1.0" 200 647168
+unicomp6.unicomp.net - - [01/Jul/1995:00:03:09 -0400] "GET /images/lcc-small2.gif HTTP/1.0" 200 58026
+gayle-gaston.tenet.edu - - [01/Jul/1995:00:03:11 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+129.188.154.200 - - [01/Jul/1995:00:03:12 -0400] "GET /facts/about_ksc.html HTTP/1.0" 200 3977
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:03:13 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+link097.txdirect.net - - [01/Jul/1995:00:03:13 -0400] "GET /shuttle/missions/sts-65/mission-sts-65.html HTTP/1.0" 200 130811
+lmsmith.tezcat.com - - [01/Jul/1995:00:03:13 -0400] "GET /cgi-bin/imagemap/countdown?103,175 HTTP/1.0" 302 110
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:03:14 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+129.188.154.200 - - [01/Jul/1995:00:03:14 -0400] "GET /images/launchpalms-small.gif HTTP/1.0" 200 11473
+lmsmith.tezcat.com - - [01/Jul/1995:00:03:14 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+brandt.xensei.com - - [01/Jul/1995:00:03:15 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+charger.execpc.com - - [01/Jul/1995:00:03:15 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+129.188.154.200 - - [01/Jul/1995:00:03:16 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+129.188.154.200 - - [01/Jul/1995:00:03:16 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+charger.execpc.com - - [01/Jul/1995:00:03:16 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+129.94.144.152 - - [01/Jul/1995:00:03:19 -0400] "GET /images/shuttle-patch-small.gif HTTP/1.0" 200 4179
+charger.execpc.com - - [01/Jul/1995:00:03:19 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+charger.execpc.com - - [01/Jul/1995:00:03:19 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+www-a1.proxy.aol.com - - [01/Jul/1995:00:03:20 -0400] "GET /cgi-bin/imagemap/countdown?107,144 HTTP/1.0" 302 96
+dd11-054.compuserve.com - - [01/Jul/1995:00:03:24 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:03:24 -0400] "GET /software HTTP/1.0" 302 -
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:03:25 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:03:26 -0400] "GET /software/ HTTP/1.0" 200 689
+port2.electrotex.com - - [01/Jul/1995:00:03:27 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:30 -0400] "GET /shuttle/missions/sts-68/sts-68-patch-small.gif HTTP/1.0" 200 17459
+midcom.com - - [01/Jul/1995:00:03:31 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:31 -0400] "GET /shuttle/missions/sts-68/mission-sts-68.html HTTP/1.0" 200 49087
+midcom.com - - [01/Jul/1995:00:03:33 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+ix-or10-06.ix.netcom.com - - [01/Jul/1995:00:03:34 -0400] "GET /software/winvn/readme.txt HTTP/1.0" 200 5736
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:36 -0400] "GET /shuttle/resources/orbiters/endeavour.html HTTP/1.0" 200 6168
+dd11-054.compuserve.com - - [01/Jul/1995:00:03:36 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:36 -0400] "GET /shuttle/resources/orbiters/endeavour-logo.gif HTTP/1.0" 200 5052
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:37 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:37 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+teleman.pr.mcs.net - - [01/Jul/1995:00:03:38 -0400] "GET /msfc/astro_home3.gif HTTP/1.0" 200 73728
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:39 -0400] "GET /shuttle/missions/sts-68/sts-68-patch-small.gif HTTP/1.0" 200 17459
+slip1.yab.com - - [01/Jul/1995:00:03:40 -0400] "GET /images/landing-747.gif HTTP/1.0" 200 110875
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:40 -0400] "GET /shuttle/missions/sts-68/mission-sts-68.html HTTP/1.0" 200 49087
+bos1d.delphi.com - - [01/Jul/1995:00:03:43 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+205.189.154.54 - - [01/Jul/1995:00:03:45 -0400] "GET /cgi-bin/imagemap/countdown?108,176 HTTP/1.0" 302 110
+charger.execpc.com - - [01/Jul/1995:00:03:47 -0400] "GET /history/history.html HTTP/1.0" 200 1502
+ix-ftw-tx1-24.ix.netcom.com - - [01/Jul/1995:00:03:48 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+midcom.com - - [01/Jul/1995:00:03:48 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+midcom.com - - [01/Jul/1995:00:03:49 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:51 -0400] "GET /facilities/lc39a.html HTTP/1.0" 200 7008
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:03:51 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0868.jpg HTTP/1.0" 200 61848
+205.189.154.54 - - [01/Jul/1995:00:03:51 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:51 -0400] "GET /images/lc39a-logo.gif HTTP/1.0" 200 13116
+port2.electrotex.com - - [01/Jul/1995:00:03:51 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 44467
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:52 -0400] "GET /images/kscmap-tiny.gif HTTP/1.0" 200 2537
+isdn6-34.dnai.com - - [01/Jul/1995:00:03:52 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+ix-ftw-tx1-24.ix.netcom.com - - [01/Jul/1995:00:03:52 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+lmsmith.tezcat.com - - [01/Jul/1995:00:03:52 -0400] "GET /shuttle/missions/sts-71/images/index71.gif HTTP/1.0" 200 73728
+charger.execpc.com - - [01/Jul/1995:00:03:53 -0400] "GET /history/apollo/images/apollo-small.gif HTTP/1.0" 200 9630
+teleman.pr.mcs.net - - [01/Jul/1995:00:03:56 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:03:57 -0400] "GET /images/launchmedium.gif HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:03:57 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:03:57 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+129.188.154.200 - - [01/Jul/1995:00:03:58 -0400] "GET / HTTP/1.0" 200 7074
+www-a1.proxy.aol.com - - [01/Jul/1995:00:04:01 -0400] "GET /cgi-bin/imagemap/countdown?99,113 HTTP/1.0" 302 111
+129.188.154.200 - - [01/Jul/1995:00:04:01 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+link097.txdirect.net - - [01/Jul/1995:00:04:02 -0400] "GET /images/shuttle-patch-small.gif HTTP/1.0" 200 4179
+smyth-pc.moorecap.com - - [01/Jul/1995:00:04:03 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+gayle-gaston.tenet.edu - - [01/Jul/1995:00:04:04 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.gif HTTP/1.0" 200 29634
+129.188.154.200 - - [01/Jul/1995:00:04:05 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+129.188.154.200 - - [01/Jul/1995:00:04:05 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+129.188.154.200 - - [01/Jul/1995:00:04:05 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+isdn6-34.dnai.com - - [01/Jul/1995:00:04:10 -0400] "GET /images/dual-pad.gif HTTP/1.0" 200 141308
+www-a1.proxy.aol.com - - [01/Jul/1995:00:04:10 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+charger.execpc.com - - [01/Jul/1995:00:04:12 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+smyth-pc.moorecap.com - - [01/Jul/1995:00:04:13 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+205.212.115.106 - - [01/Jul/1995:00:04:13 -0400] "GET /shuttle/missions/sts-71/images/index71.gif HTTP/1.0" 200 235862
+charger.execpc.com - - [01/Jul/1995:00:04:14 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+charger.execpc.com - - [01/Jul/1995:00:04:15 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+unicomp6.unicomp.net - - [01/Jul/1995:00:04:16 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+205.189.154.54 - - [01/Jul/1995:00:04:17 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0747.gif HTTP/1.0" 200 32187
+dd11-054.compuserve.com - - [01/Jul/1995:00:04:17 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+news.ti.com - - [01/Jul/1995:00:04:18 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ix-ftw-tx1-24.ix.netcom.com - - [01/Jul/1995:00:04:18 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+unicomp6.unicomp.net - - [01/Jul/1995:00:04:19 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+charger.execpc.com - - [01/Jul/1995:00:04:20 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+news.ti.com - - [01/Jul/1995:00:04:21 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ix-ftw-tx1-24.ix.netcom.com - - [01/Jul/1995:00:04:22 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+teleman.pr.mcs.net - - [01/Jul/1995:00:04:23 -0400] "GET /shuttle/missions/sts-67/mission-sts-67.html HTTP/1.0" 200 21408
+129.94.144.152 - - [01/Jul/1995:00:04:25 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 84907
+link097.txdirect.net - - [01/Jul/1995:00:04:26 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 84907
+unicomp6.unicomp.net - - [01/Jul/1995:00:04:26 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+unicomp6.unicomp.net - - [01/Jul/1995:00:04:26 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+onyx.southwind.net - - [01/Jul/1995:00:04:27 -0400] "GET /shuttle/technology/sts-newsref/centers.html HTTP/1.0" 200 71024
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:04:27 -0400] "GET /shuttle/missions/sts-69/mission-sts-69.html HTTP/1.0" 200 4325
+unicomp6.unicomp.net - - [01/Jul/1995:00:04:27 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+129.188.154.200 - - [01/Jul/1995:00:04:27 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+netport-27.iu.net - - [01/Jul/1995:00:04:27 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+slip1.yab.com - - [01/Jul/1995:00:04:28 -0400] "GET /history/history.html HTTP/1.0" 200 1502
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:04:28 -0400] "GET /shuttle/missions/sts-69/sts-69-patch-small.gif HTTP/1.0" 200 8083
+teleman.pr.mcs.net - - [01/Jul/1995:00:04:30 -0400] "GET /shuttle/missions/sts-67/sts-67-patch-small.gif HTTP/1.0" 200 17083
+129.188.154.200 - - [01/Jul/1995:00:04:30 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+savvy1.savvy.com - - [01/Jul/1995:00:04:30 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+gater3.sematech.org - - [01/Jul/1995:00:04:31 -0400] "GET /cgi-bin/imagemap/countdown?370,274 HTTP/1.0" 302 68
+teleman.pr.mcs.net - - [01/Jul/1995:00:04:31 -0400] "GET /images/launch-logo.gif HTTP/1.0" 304 0
+slip1.yab.com - - [01/Jul/1995:00:04:32 -0400] "GET /history/rocket-history.txt HTTP/1.0" 200 26990
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:04:33 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-hatch-hand-group.mpg HTTP/1.0" 200 49152
+netport-27.iu.net - - [01/Jul/1995:00:04:34 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+savvy1.savvy.com - - [01/Jul/1995:00:04:34 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+199.33.32.50 - - [01/Jul/1995:00:04:36 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-launch-3.mpg HTTP/1.0" 200 130724
+129.188.154.200 - - [01/Jul/1995:00:04:37 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+pme607.onramp.awinc.com - - [01/Jul/1995:00:04:38 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+bart-slip1.llnl.gov - - [01/Jul/1995:00:04:40 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 84907
+ad03-031.compuserve.com - - [01/Jul/1995:00:04:40 -0400] "GET /facilities/vab.html HTTP/1.0" 200 4045
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:04:44 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+savvy1.savvy.com - - [01/Jul/1995:00:04:44 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+savvy1.savvy.com - - [01/Jul/1995:00:04:44 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:04:46 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:04:46 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+news.ti.com - - [01/Jul/1995:00:04:46 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:04:48 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:04:48 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:04:50 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+ad03-031.compuserve.com - - [01/Jul/1995:00:04:51 -0400] "GET /images/vab-small.gif HTTP/1.0" 200 35709
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:04:51 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+199.166.39.14 - - [01/Jul/1995:00:04:52 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+news.ti.com - - [01/Jul/1995:00:04:52 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+pme607.onramp.awinc.com - - [01/Jul/1995:00:04:54 -0400] "GET /cgi-bin/imagemap/countdown?99,178 HTTP/1.0" 302 110
+news.ti.com - - [01/Jul/1995:00:04:55 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:04:55 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0871.gif HTTP/1.0" 200 45518
+charger.execpc.com - - [01/Jul/1995:00:04:55 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+199.166.39.14 - - [01/Jul/1995:00:04:56 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+onyx.southwind.net - - [01/Jul/1995:00:04:58 -0400] "GET /images/shuttle-patch-logo.gif HTTP/1.0" 200 891
+199.166.39.14 - - [01/Jul/1995:00:04:59 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+pme607.onramp.awinc.com - - [01/Jul/1995:00:05:02 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:05:06 -0400] "GET /cgi-bin/imagemap/countdown?107,144 HTTP/1.0" 302 96
+link097.txdirect.net - - [01/Jul/1995:00:05:06 -0400] "GET /shuttle HTTP/1.0" 302 -
+link097.txdirect.net - - [01/Jul/1995:00:05:07 -0400] "GET /shuttle/ HTTP/1.0" 200 957
+dd11-054.compuserve.com - - [01/Jul/1995:00:05:09 -0400] "GET /shuttle/missions/61-c/mission-61-c.html HTTP/1.0" 200 8368
+charger.execpc.com - - [01/Jul/1995:00:05:12 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:13 -0400] "GET /shuttle/missions/sts-78/news HTTP/1.0" 302 -
+link097.txdirect.net - - [01/Jul/1995:00:05:13 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:14 -0400] "GET /shuttle/missions/sts-78/news/ HTTP/1.0" 200 374
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:15 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:17 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+net-1-141.eden.com - - [01/Jul/1995:00:05:17 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+teleman.pr.mcs.net - - [01/Jul/1995:00:05:18 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+199.166.39.14 - - [01/Jul/1995:00:05:18 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+waters-gw.starway.net.au - - [01/Jul/1995:00:05:19 -0400] "GET /shuttle/missions/51-l/images/850128.GIF HTTP/1.0" 200 172498
+teleman.pr.mcs.net - - [01/Jul/1995:00:05:20 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 304 0
+199.166.39.14 - - [01/Jul/1995:00:05:21 -0400] "GET /cgi-bin/imagemap/countdown?302,275 HTTP/1.0" 302 98
+slip132.indirect.com - - [01/Jul/1995:00:05:22 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:23 -0400] "GET /shuttle/missions/sts-78/ HTTP/1.0" 200 1596
+link097.txdirect.net - - [01/Jul/1995:00:05:23 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:23 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:24 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+slip1.yab.com - - [01/Jul/1995:00:05:24 -0400] "GET /history/early-astronauts.txt HTTP/1.0" 200 3850
+www-a1.proxy.aol.com - - [01/Jul/1995:00:05:25 -0400] "GET / HTTP/1.0" 200 7074
+199.166.39.14 - - [01/Jul/1995:00:05:26 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+link097.txdirect.net - - [01/Jul/1995:00:05:29 -0400] "GET /shuttle/movies/ HTTP/1.0" 200 602
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:05:30 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+link097.txdirect.net - - [01/Jul/1995:00:05:30 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:05:31 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+129.94.144.152 - - [01/Jul/1995:00:05:35 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+port2.electrotex.com - - [01/Jul/1995:00:05:37 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 43974
+129.94.144.152 - - [01/Jul/1995:00:05:37 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:05:37 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+link097.txdirect.net - - [01/Jul/1995:00:05:38 -0400] "GET /shuttle/movies/astronauts.mpg HTTP/1.0" 200 49152
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:05:39 -0400] "GET /shuttle/missions/sts-71/sts-71-patch.jpg HTTP/1.0" 200 203429
+charger.execpc.com - - [01/Jul/1995:00:05:39 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+205.212.115.106 - - [01/Jul/1995:00:05:42 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+alyssa.prodigy.com - - [01/Jul/1995:00:05:43 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:05:43 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0876.gif HTTP/1.0" 200 51398
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:43 -0400] "GET /shuttle/missions/sts-72/mission-sts-72.html HTTP/1.0" 200 3752
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:45 -0400] "GET /shuttle/missions/sts-72/sts-72-patch-small.gif HTTP/1.0" 200 4179
+teleman.pr.mcs.net - - [01/Jul/1995:00:05:46 -0400] "GET /shuttle/missions/sts-70/mission-sts-70.html HTTP/1.0" 200 13469
+teleman.pr.mcs.net - - [01/Jul/1995:00:05:48 -0400] "GET /shuttle/missions/sts-70/sts-70-patch-small.gif HTTP/1.0" 200 5026
+slip1.yab.com - - [01/Jul/1995:00:05:53 -0400] "GET /history/skylab/skylab.html HTTP/1.0" 200 1687
+199.166.39.14 - - [01/Jul/1995:00:05:53 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 65536
+129.188.154.200 - - [01/Jul/1995:00:05:53 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:05:54 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip4068.sirius.com - - [01/Jul/1995:00:05:55 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+pm4_23.digital.net - - [01/Jul/1995:00:05:58 -0400] "GET /images/shuttle-patch-small.gif HTTP/1.0" 200 4179
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:05:58 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:58 -0400] "GET /shuttle/missions/sts-72/news HTTP/1.0" 302 -
+mars.ark.com - - [01/Jul/1995:00:05:59 -0400] "GET /facts/faq04.html HTTP/1.0" 200 27063
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:05:59 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:05:59 -0400] "GET /shuttle/missions/sts-72/news/ HTTP/1.0" 200 374
+news.ti.com - - [01/Jul/1995:00:06:00 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+janice.cc.wwu.edu - - [01/Jul/1995:00:06:01 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+dd11-054.compuserve.com - - [01/Jul/1995:00:06:02 -0400] "GET /history/history.html HTTP/1.0" 200 1502
+news.ti.com - - [01/Jul/1995:00:06:03 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+gayle-gaston.tenet.edu - - [01/Jul/1995:00:06:03 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:06:05 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dd11-054.compuserve.com - - [01/Jul/1995:00:06:05 -0400] "GET /history/apollo/images/apollo-small.gif HTTP/1.0" 200 9630
+dnet018.sat.texas.net - - [01/Jul/1995:00:06:06 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+teleman.pr.mcs.net - - [01/Jul/1995:00:06:06 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+teleman.pr.mcs.net - - [01/Jul/1995:00:06:08 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+mars.ark.com - - [01/Jul/1995:00:06:11 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+mars.ark.com - - [01/Jul/1995:00:06:11 -0400] "GET /images/faq.gif HTTP/1.0" 200 263
+burger.letters.com - - [01/Jul/1995:00:06:12 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+burger.letters.com - - [01/Jul/1995:00:06:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 0
+burger.letters.com - - [01/Jul/1995:00:06:13 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+net-1-141.eden.com - - [01/Jul/1995:00:06:15 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+ad12-051.compuserve.com - - [01/Jul/1995:00:06:15 -0400] "GET /history/history.html HTTP/1.0" 200 1502
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:06:17 -0400] "GET /history/apollo/apollo-11/apollo-11-info.html HTTP/1.0" 200 1457
+www-a1.proxy.aol.com - - [01/Jul/1995:00:06:19 -0400] "GET /whats-new.html HTTP/1.0" 200 17314
+ad12-051.compuserve.com - - [01/Jul/1995:00:06:19 -0400] "GET /history/apollo/images/apollo-small.gif HTTP/1.0" 200 9630
+citynet.ci.la.ca.us - - [01/Jul/1995:00:06:20 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+gater4.sematech.org - - [01/Jul/1995:00:06:26 -0400] "GET /cgi-bin/imagemap/countdown?88,208 HTTP/1.0" 302 95
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:06:26 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:06:27 -0400] "GET /facilities/lc39a.html HTTP/1.0" 200 7008
+gater3.sematech.org - - [01/Jul/1995:00:06:28 -0400] "GET /shuttle/countdown/tour.html HTTP/1.0" 200 4347
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:06:28 -0400] "GET /images/lc39a-logo.gif HTTP/1.0" 200 13116
+ottgate2.bnr.ca - - [01/Jul/1995:00:06:28 -0400] "GET /shuttle/technology/images/srb_mod_compare_1-small.gif HTTP/1.0" 200 36902
+ad12-051.compuserve.com - - [01/Jul/1995:00:06:30 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ottgate2.bnr.ca - - [01/Jul/1995:00:06:30 -0400] "GET /shuttle/technology/images/srb_mod_compare_6-small.gif HTTP/1.0" 200 28219
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:06:32 -0400] "GET /images/kscmap-tiny.gif HTTP/1.0" 200 2537
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:06:33 -0400] "GET /history/apollo/apollo-11/images/ HTTP/1.0" 200 3459
+ix-war-mi1-20.ix.netcom.com - - [01/Jul/1995:00:06:34 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+janice.cc.wwu.edu - - [01/Jul/1995:00:06:35 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+news.ti.com - - [01/Jul/1995:00:06:36 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:06:36 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+pme607.onramp.awinc.com - - [01/Jul/1995:00:06:36 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+ottgate2.bnr.ca - - [01/Jul/1995:00:06:36 -0400] "GET /shuttle/technology/images/srb_mod_compare_3-small.gif HTTP/1.0" 200 55666
+gater3.sematech.org - - [01/Jul/1995:00:06:37 -0400] "GET /images/KSC-94EC-412-small.gif HTTP/1.0" 200 20484
+teleman.pr.mcs.net - - [01/Jul/1995:00:06:39 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+www-a1.proxy.aol.com - - [01/Jul/1995:00:06:39 -0400] "GET /images/whatsnew.gif HTTP/1.0" 200 651
+slip132.indirect.com - - [01/Jul/1995:00:06:41 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+teleman.pr.mcs.net - - [01/Jul/1995:00:06:41 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+alyssa.prodigy.com - - [01/Jul/1995:00:06:42 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+charger.execpc.com - - [01/Jul/1995:00:06:43 -0400] "GET /history/mercury/mercury.html HTTP/1.0" 200 1871
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:06:46 -0400] "GET /history/apollo/apollo-13/movies/apo13inside.mpg HTTP/1.0" 200 501126
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:06:46 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+janice.cc.wwu.edu - - [01/Jul/1995:00:06:48 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 48725
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:06:48 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+charger.execpc.com - - [01/Jul/1995:00:06:49 -0400] "GET /images/mercury-logo.gif HTTP/1.0" 200 6588
+brandt.xensei.com - - [01/Jul/1995:00:06:54 -0400] "GET /shuttle/missions/sts-71/sts-71-day-04-highlights.html HTTP/1.0" 200 5544
+alyssa.prodigy.com - - [01/Jul/1995:00:06:55 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:06:59 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:07:00 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:07:01 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:07:02 -0400] "GET /cgi-bin/imagemap/countdown?98,110 HTTP/1.0" 302 111
+slip1.yab.com - - [01/Jul/1995:00:07:03 -0400] "GET /history/skylab/skylab.jpg HTTP/1.0" 200 162605
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:07:04 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:07:04 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:07:08 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+gater3.sematech.org - - [01/Jul/1995:00:07:13 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+pm4_23.digital.net - - [01/Jul/1995:00:07:13 -0400] "GET /shuttle/technology/sts-newsref/sts_egress.html HTTP/1.0" 200 86381
+dd11-054.compuserve.com - - [01/Jul/1995:00:07:14 -0400] "GET /shuttle/missions/sts-67/mission-sts-67.html HTTP/1.0" 200 21408
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:07:15 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+gater3.sematech.org - - [01/Jul/1995:00:07:16 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+waters-gw.starway.net.au - - [01/Jul/1995:00:07:19 -0400] "GET /shuttle/missions/51-l/images/86HC119.GIF HTTP/1.0" 200 127775
+205.212.115.106 - - [01/Jul/1995:00:07:20 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+ppp236.iadfw.net - - [01/Jul/1995:00:07:20 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ad12-051.compuserve.com - - [01/Jul/1995:00:07:20 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+charger.execpc.com - - [01/Jul/1995:00:07:21 -0400] "GET /history/mercury/mr-3/mr-3.html HTTP/1.0" 200 1124
+ppp236.iadfw.net - - [01/Jul/1995:00:07:21 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:07:21 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+charger.execpc.com - - [01/Jul/1995:00:07:22 -0400] "GET /history/mercury/mr-3/mr-3-patch-small.gif HTTP/1.0" 200 19084
+asp.erinet.com - - [01/Jul/1995:00:07:23 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ppp236.iadfw.net - - [01/Jul/1995:00:07:26 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+ppp236.iadfw.net - - [01/Jul/1995:00:07:26 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+asp.erinet.com - - [01/Jul/1995:00:07:26 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+asp.erinet.com - - [01/Jul/1995:00:07:27 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+ppp236.iadfw.net - - [01/Jul/1995:00:07:27 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+charger.execpc.com - - [01/Jul/1995:00:07:29 -0400] "GET /history/apollo/images/APOLLO-logosmall.gif HTTP/1.0" 200 1173
+asp.erinet.com - - [01/Jul/1995:00:07:29 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+ppp236.iadfw.net - - [01/Jul/1995:00:07:29 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+asp.erinet.com - - [01/Jul/1995:00:07:30 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+asp.erinet.com - - [01/Jul/1995:00:07:30 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+asp.erinet.com - - [01/Jul/1995:00:07:31 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:31 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+www-proxy.crl.research.digital.com - - [01/Jul/1995:00:07:33 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:34 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:34 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:34 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+asp.erinet.com - - [01/Jul/1995:00:07:34 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+news.ti.com - - [01/Jul/1995:00:07:34 -0400] "GET /shuttle/missions/sts-71/sts-71-day-04-highlights.html HTTP/1.0" 200 5544
+teleman.pr.mcs.net - - [01/Jul/1995:00:07:35 -0400] "GET /cgi-bin/imagemap/countdown?102,172 HTTP/1.0" 302 110
+www-proxy.crl.research.digital.com - - [01/Jul/1995:00:07:35 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+teleman.pr.mcs.net - - [01/Jul/1995:00:07:36 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ppp236.iadfw.net - - [01/Jul/1995:00:07:36 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ppp4.sunrem.com - - [01/Jul/1995:00:07:37 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+199.166.39.14 - - [01/Jul/1995:00:07:37 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-tcdt-crew-walkout.mpg HTTP/1.0" 200 131072
+ppp236.iadfw.net - - [01/Jul/1995:00:07:37 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+slip132.indirect.com - - [01/Jul/1995:00:07:37 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+ppp236.iadfw.net - - [01/Jul/1995:00:07:37 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ppp4.sunrem.com - - [01/Jul/1995:00:07:39 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+pm13.j51.com - - [01/Jul/1995:00:07:40 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 65536
+ppp4.sunrem.com - - [01/Jul/1995:00:07:41 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:42 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ppp4.sunrem.com - - [01/Jul/1995:00:07:43 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+pm4_23.digital.net - - [01/Jul/1995:00:07:43 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+pm4_23.digital.net - - [01/Jul/1995:00:07:43 -0400] "GET /images/shuttle-patch-logo.gif HTTP/1.0" 200 891
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:43 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+charger.execpc.com - - [01/Jul/1995:00:07:44 -0400] "GET /history/mercury/ma-6/ma-6.html HTTP/1.0" 200 1117
+citynet.ci.la.ca.us - - [01/Jul/1995:00:07:45 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+charger.execpc.com - - [01/Jul/1995:00:07:46 -0400] "GET /history/mercury/ma-6/ma-6-patch-small.gif HTTP/1.0" 200 21277
+dnet018.sat.texas.net - - [01/Jul/1995:00:07:47 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+dnet018.sat.texas.net - - [01/Jul/1995:00:07:50 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:07:50 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+asp.erinet.com - - [01/Jul/1995:00:07:52 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:07:52 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:07:53 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+dnet018.sat.texas.net - - [01/Jul/1995:00:07:56 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+slip1.yab.com - - [01/Jul/1995:00:07:57 -0400] "GET /history/skylab/skylab-2.html HTTP/1.0" 200 1708
+dnet018.sat.texas.net - - [01/Jul/1995:00:07:59 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+dnet018.sat.texas.net - - [01/Jul/1995:00:08:00 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+166.79.67.111 - - [01/Jul/1995:00:08:02 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:08:03 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:08:03 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dnet018.sat.texas.net - - [01/Jul/1995:00:08:05 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+www-proxy.crl.research.digital.com - - [01/Jul/1995:00:08:05 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+brandt.xensei.com - - [01/Jul/1995:00:08:06 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:08:07 -0400] "GET /history/apollo/apollo-11/images/index.gif HTTP/1.0" 200 172537
+www-proxy.crl.research.digital.com - - [01/Jul/1995:00:08:09 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 75100
+slip1.kias.com - - [01/Jul/1995:00:08:10 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+ix-li3-23.ix.netcom.com - - [01/Jul/1995:00:08:11 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:08:12 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+brandt.xensei.com - - [01/Jul/1995:00:08:13 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+net-1-141.eden.com - - [01/Jul/1995:00:08:13 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0613.jpg HTTP/1.0" 200 61716
+166.79.67.111 - - [01/Jul/1995:00:08:21 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:08:22 -0400] "GET /cgi-bin/imagemap/countdown?382,274 HTTP/1.0" 302 68
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:22 -0400] "GET / HTTP/1.0" 200 7074
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:25 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+166.79.67.111 - - [01/Jul/1995:00:08:26 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+slip1.kias.com - - [01/Jul/1995:00:08:26 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+slip1.kias.com - - [01/Jul/1995:00:08:27 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:27 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:27 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+199.120.91.6 - - [01/Jul/1995:00:08:28 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+129.94.144.152 - - [01/Jul/1995:00:08:29 -0400] "GET /shuttle/technology/sts-newsref/sts-cron.html HTTP/1.0" 200 200053
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:08:30 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:31 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ttyu0.tyrell.net - - [01/Jul/1995:00:08:31 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+slip132.indirect.com - - [01/Jul/1995:00:08:31 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.txt HTTP/1.0" 200 1391
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:08:32 -0400] "GET /cgi-bin/imagemap/countdown?94,175 HTTP/1.0" 302 110
+199.120.91.6 - - [01/Jul/1995:00:08:32 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+199.120.91.6 - - [01/Jul/1995:00:08:32 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:08:33 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+slip1.yab.com - - [01/Jul/1995:00:08:38 -0400] "GET /history/skylab/skylab-3.html HTTP/1.0" 200 1424
+dnet018.sat.texas.net - - [01/Jul/1995:00:08:41 -0400] "GET /whats-new.html HTTP/1.0" 200 17314
+199.120.91.6 - - [01/Jul/1995:00:08:43 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dnet018.sat.texas.net - - [01/Jul/1995:00:08:43 -0400] "GET /images/whatsnew.gif HTTP/1.0" 200 651
+ppp4.sunrem.com - - [01/Jul/1995:00:08:45 -0400] "GET /cgi-bin/imagemap/countdown?105,143 HTTP/1.0" 302 96
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:08:49 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip1.kias.com - - [01/Jul/1995:00:08:49 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+brandt.xensei.com - - [01/Jul/1995:00:08:50 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 75344
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:08:52 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:08:52 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:08:52 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dnet018.sat.texas.net - - [01/Jul/1995:00:08:54 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+www-a1.proxy.aol.com - - [01/Jul/1995:00:08:56 -0400] "GET /cgi-bin/imagemap/countdown?99,213 HTTP/1.0" 302 95
+www-a1.proxy.aol.com - - [01/Jul/1995:00:08:59 -0400] "GET /shuttle/countdown/tour.html HTTP/1.0" 200 4347
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:09:00 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:09:00 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0869.jpg HTTP/1.0" 200 56275
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:09:00 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:09:01 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+slip1.yab.com - - [01/Jul/1995:00:09:02 -0400] "GET /history/skylab/skylab-4.html HTTP/1.0" 200 1393
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:09:02 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:09:03 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+129.94.144.152 - - [01/Jul/1995:00:09:05 -0400] "GET /images/shuttle-patch-logo.gif HTTP/1.0" 200 891
+129.94.144.152 - - [01/Jul/1995:00:09:06 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+teleman.pr.mcs.net - - [01/Jul/1995:00:09:07 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0423.jpg HTTP/1.0" 200 84494
+piweba3y.prodigy.com - - [01/Jul/1995:00:09:07 -0400] "GET /shuttle/technology/images/srb_mod_compare_6-small.gif HTTP/1.0" 200 28219
+waters-gw.starway.net.au - - [01/Jul/1995:00:09:16 -0400] "GET /shuttle/missions/51-l/images/86HC125.GIF HTTP/1.0" 200 100705
+www-a1.proxy.aol.com - - [01/Jul/1995:00:09:16 -0400] "GET /images/KSC-94EC-412-small.gif HTTP/1.0" 200 20484
+166.79.67.111 - - [01/Jul/1995:00:09:17 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+wwwproxy.info.au - - [01/Jul/1995:00:09:18 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:09:18 -0400] "GET /cgi-bin/imagemap/countdown?99,150 HTTP/1.0" 302 96
+news.ti.com - - [01/Jul/1995:00:09:19 -0400] "GET /shuttle/missions/sts-71/sts-71-day-03-highlights.html HTTP/1.0" 200 6104
+wwwproxy.info.au - - [01/Jul/1995:00:09:21 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+wwwproxy.info.au - - [01/Jul/1995:00:09:21 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+citynet.ci.la.ca.us - - [01/Jul/1995:00:09:23 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+199.120.91.6 - - [01/Jul/1995:00:09:25 -0400] "GET /cgi-bin/imagemap/countdown?373,278 HTTP/1.0" 302 68
+wwwproxy.info.au - - [01/Jul/1995:00:09:25 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+wwwproxy.info.au - - [01/Jul/1995:00:09:28 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+pm110.spectra.net - - [01/Jul/1995:00:09:30 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+wwwproxy.info.au - - [01/Jul/1995:00:09:30 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+pm110.spectra.net - - [01/Jul/1995:00:09:33 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+brandt.xensei.com - - [01/Jul/1995:00:09:34 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+wwwproxy.info.au - - [01/Jul/1995:00:09:35 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 304 0
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:09:36 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+129.94.144.152 - - [01/Jul/1995:00:09:37 -0400] "GET /shuttle/missions/51-l/mission-51-l.html HTTP/1.0" 200 6723
+pm110.spectra.net - - [01/Jul/1995:00:09:37 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+wwwproxy.info.au - - [01/Jul/1995:00:09:39 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+wwwproxy.info.au - - [01/Jul/1995:00:09:39 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 304 0
+pm110.spectra.net - - [01/Jul/1995:00:09:40 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+dnet018.sat.texas.net - - [01/Jul/1995:00:09:44 -0400] "GET /shuttle/missions/sts-63/sts-63-patch-small.gif HTTP/1.0" 200 16102
+brandt.xensei.com - - [01/Jul/1995:00:09:46 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+net-1-141.eden.com - - [01/Jul/1995:00:09:47 -0400] "GET /cgi-bin/imagemap/countdown?376,273 HTTP/1.0" 302 68
+teleman.pr.mcs.net - - [01/Jul/1995:00:09:48 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0613.jpg HTTP/1.0" 200 61716
+129.94.144.152 - - [01/Jul/1995:00:09:50 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+129.94.144.152 - - [01/Jul/1995:00:09:52 -0400] "GET /shuttle/missions/51-l/51-l-patch-small.gif HTTP/1.0" 200 10495
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:09:53 -0400] "GET /cgi-bin/imagemap/countdown?94,145 HTTP/1.0" 302 96
+pm110.spectra.net - - [01/Jul/1995:00:09:54 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip1.kias.com - - [01/Jul/1995:00:09:55 -0400] "GET /htbin/wais.pl HTTP/1.0" 200 308
+pm110.spectra.net - - [01/Jul/1995:00:09:56 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dnet018.sat.texas.net - - [01/Jul/1995:00:09:57 -0400] "GET /shuttle/missions/sts-63/mission-sts-63.html HTTP/1.0" 200 53003
+pm110.spectra.net - - [01/Jul/1995:00:09:57 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:10:01 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+205.212.115.106 - - [01/Jul/1995:00:10:01 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0876.gif HTTP/1.0" 200 51398
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:10:02 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+129.188.154.200 - - [01/Jul/1995:00:10:04 -0400] "GET /shuttle/missions/sts-71/movies/ HTTP/1.0" 200 3090
+129.188.154.200 - - [01/Jul/1995:00:10:07 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+129.188.154.200 - - [01/Jul/1995:00:10:07 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+129.188.154.200 - - [01/Jul/1995:00:10:07 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+129.188.154.200 - - [01/Jul/1995:00:10:07 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+citynet.ci.la.ca.us - - [01/Jul/1995:00:10:08 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+brandt.xensei.com - - [01/Jul/1995:00:10:08 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 74691
+slip1.kias.com - - [01/Jul/1995:00:10:12 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+slip1.kias.com - - [01/Jul/1995:00:10:15 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+www-a1.proxy.aol.com - - [01/Jul/1995:00:10:16 -0400] "GET /shuttle/countdown/lps/fr.html HTTP/1.0" 200 1879
+netport-27.iu.net - - [01/Jul/1995:00:10:19 -0400] "GET /pub/winvn/readme.txt HTTP/1.0" 404 -
+corp-uu.infoseek.com - - [01/Jul/1995:00:10:22 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+129.188.154.200 - - [01/Jul/1995:00:10:23 -0400] "GET /shuttle/missions/sts-71/ HTTP/1.0" 200 3373
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:10:23 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:10:24 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.jpg HTTP/1.0" 200 34029
+netport-27.iu.net - - [01/Jul/1995:00:10:28 -0400] "GET /pub/winvn/readme.txt HTTP/1.0" 404 -
+slip1.kias.com - - [01/Jul/1995:00:10:28 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+205.157.131.144 - - [01/Jul/1995:00:10:28 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+129.188.154.200 - - [01/Jul/1995:00:10:29 -0400] "GET /shuttle/missions/sts-71/images/ HTTP/1.0" 200 16890
+www-a1.proxy.aol.com - - [01/Jul/1995:00:10:29 -0400] "GET /shuttle/countdown/lps/back.gif HTTP/1.0" 200 1289
+129.188.154.200 - - [01/Jul/1995:00:10:29 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+waters-gw.starway.net.au - - [01/Jul/1995:00:10:31 -0400] "GET /shuttle/missions/51-l/images/86HC159.GIF HTTP/1.0" 200 78295
+www-d3.proxy.aol.com - - [01/Jul/1995:00:10:32 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ppp4.sunrem.com - - [01/Jul/1995:00:10:33 -0400] "GET /cgi-bin/imagemap/countdown?93,176 HTTP/1.0" 302 110
+ppp4.sunrem.com - - [01/Jul/1995:00:10:34 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+205.157.131.144 - - [01/Jul/1995:00:10:34 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+205.157.131.144 - - [01/Jul/1995:00:10:34 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+205.157.131.144 - - [01/Jul/1995:00:10:36 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+dnet018.sat.texas.net - - [01/Jul/1995:00:10:37 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:10:38 -0400] "GET /cgi-bin/imagemap/countdown?103,180 HTTP/1.0" 302 110
+129.94.144.152 - - [01/Jul/1995:00:10:39 -0400] "GET /shuttle/missions/51-l/51-l-info.html HTTP/1.0" 200 1387
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:10:40 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+brandt.xensei.com - - [01/Jul/1995:00:10:41 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+p06.eznets.canton.oh.us - - [01/Jul/1995:00:10:43 -0400] "GET /cgi-bin/imagemap/countdown?382,271 HTTP/1.0" 302 68
+brandt.xensei.com - - [01/Jul/1995:00:10:43 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:10:43 -0400] "GET / HTTP/1.0" 200 7074
+taiki4.envi.osakafu-u.ac.jp - - [01/Jul/1995:00:10:45 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+taiki4.envi.osakafu-u.ac.jp - - [01/Jul/1995:00:10:48 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:10:49 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+dynip38.efn.org - - [01/Jul/1995:00:10:50 -0400] "GET /software HTTP/1.0" 302 -
+dynip38.efn.org - - [01/Jul/1995:00:10:51 -0400] "GET /software/ HTTP/1.0" 200 689
+129.188.154.200 - - [01/Jul/1995:00:10:52 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+teleman.pr.mcs.net - - [01/Jul/1995:00:10:53 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0871.jpg HTTP/1.0" 200 60280
+dynip38.efn.org - - [01/Jul/1995:00:10:53 -0400] "GET /icons/blank.xbm HTTP/1.0" 304 0
+dynip38.efn.org - - [01/Jul/1995:00:10:53 -0400] "GET /icons/menu.xbm HTTP/1.0" 304 0
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:10:55 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:10:57 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 75142
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:10:59 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:11:00 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:11:02 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+dynip38.efn.org - - [01/Jul/1995:00:11:02 -0400] "GET /software/winvn/ HTTP/1.0" 200 2244
+taiki4.envi.osakafu-u.ac.jp - - [01/Jul/1995:00:11:03 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dynip38.efn.org - - [01/Jul/1995:00:11:04 -0400] "GET /icons/text.xbm HTTP/1.0" 304 0
+133.127.203.203 - - [01/Jul/1995:00:11:08 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+dynip38.efn.org - - [01/Jul/1995:00:11:09 -0400] "GET /icons/image.xbm HTTP/1.0" 304 0
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:11:12 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.jpg HTTP/1.0" 200 45966
+brandt.xensei.com - - [01/Jul/1995:00:11:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 75142
+dnet018.sat.texas.net - - [01/Jul/1995:00:11:13 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:11:13 -0400] "GET /shuttle/missions/sts-71/images/index71.gif HTTP/1.0" 200 49152
+129.188.154.200 - - [01/Jul/1995:00:11:14 -0400] "GET /shuttle/missions HTTP/1.0" 302 -
+ottgate2.bnr.ca - - [01/Jul/1995:00:11:14 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+129.188.154.200 - - [01/Jul/1995:00:11:15 -0400] "GET /shuttle/missions/ HTTP/1.0" 200 12283
+www-d3.proxy.aol.com - - [01/Jul/1995:00:11:16 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+dd11-054.compuserve.com - - [01/Jul/1995:00:11:17 -0400] "GET /shuttle/missions/sts-67/sts-67-patch-small.gif HTTP/1.0" 200 17083
+slip1.kias.com - - [01/Jul/1995:00:11:19 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+news.ti.com - - [01/Jul/1995:00:11:19 -0400] "GET /shuttle/missions/100th.html HTTP/1.0" 200 32303
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:21 -0400] "GET / HTTP/1.0" 200 7074
+204.19.123.36 - - [01/Jul/1995:00:11:23 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ad11-010.compuserve.com - - [01/Jul/1995:00:11:24 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+www-a1.proxy.aol.com - - [01/Jul/1995:00:11:24 -0400] "GET /facilities/tour.html HTTP/1.0" 200 3723
+slip1.kias.com - - [01/Jul/1995:00:11:24 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+teleman.pr.mcs.net - - [01/Jul/1995:00:11:26 -0400] "GET /cgi-bin/imagemap/countdown?107,147 HTTP/1.0" 302 96
+205.212.115.106 - - [01/Jul/1995:00:11:27 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.gif HTTP/1.0" 200 31242
+156.151.176.30 - - [01/Jul/1995:00:11:27 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+pm28.sonic.net - - [01/Jul/1995:00:11:27 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+205.157.131.144 - - [01/Jul/1995:00:11:27 -0400] "GET /cgi-bin/imagemap/countdown?92,174 HTTP/1.0" 302 110
+205.157.131.144 - - [01/Jul/1995:00:11:28 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:29 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+dnet018.sat.texas.net - - [01/Jul/1995:00:11:29 -0400] "GET /history/apollo/apollo-13/movies/ HTTP/1.0" 200 945
+dnet018.sat.texas.net - - [01/Jul/1995:00:11:30 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+pm28.sonic.net - - [01/Jul/1995:00:11:31 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:11:32 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+piweba1y.prodigy.com - - [01/Jul/1995:00:11:33 -0400] "GET / HTTP/1.0" 200 7074
+pm28.sonic.net - - [01/Jul/1995:00:11:34 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dnet018.sat.texas.net - - [01/Jul/1995:00:11:35 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+citynet.ci.la.ca.us - - [01/Jul/1995:00:11:35 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0443.gif HTTP/1.0" 200 64910
+dnet018.sat.texas.net - - [01/Jul/1995:00:11:35 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+129.188.154.200 - - [01/Jul/1995:00:11:35 -0400] "GET /shuttle HTTP/1.0" 302 -
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:11:36 -0400] "GET /history/apollo/apollo-11/images/690700.GIF HTTP/1.0" 200 125771
+dynip38.efn.org - - [01/Jul/1995:00:11:36 -0400] "GET /software/winvn/release.txt HTTP/1.0" 200 23052
+pm28.sonic.net - - [01/Jul/1995:00:11:36 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:11:36 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ppp4.sunrem.com - - [01/Jul/1995:00:11:37 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0443.gif HTTP/1.0" 200 64910
+129.188.154.200 - - [01/Jul/1995:00:11:37 -0400] "GET /shuttle/ HTTP/1.0" 200 957
+204.120.76.119 - - [01/Jul/1995:00:11:38 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+129.188.154.200 - - [01/Jul/1995:00:11:38 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-launch.mpg HTTP/1.0" 200 106496
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:40 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+www-d3.proxy.aol.com - - [01/Jul/1995:00:11:41 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.gif HTTP/1.0" 200 29634
+ottgate2.bnr.ca - - [01/Jul/1995:00:11:42 -0400] "GET /shuttle/countdown/count.html HTTP/1.0" 200 73231
+ix-dfw13-20.ix.netcom.com - - [01/Jul/1995:00:11:44 -0400] "GET /cgi-bin/imagemap/countdown?95,142 HTTP/1.0" 302 96
+leet.cts.com - - [01/Jul/1995:00:11:45 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:11:46 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0906.jpg HTTP/1.0" 200 58414
+pm110.spectra.net - - [01/Jul/1995:00:11:46 -0400] "GET /cgi-bin/imagemap/countdown?100,114 HTTP/1.0" 302 111
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:47 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+129.188.154.200 - - [01/Jul/1995:00:11:48 -0400] "GET /shuttle/movies/ HTTP/1.0" 200 602
+200.13.2.37 - - [01/Jul/1995:00:11:48 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+dynip38.efn.org - - [01/Jul/1995:00:11:49 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:11:49 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:49 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+dynip38.efn.org - - [01/Jul/1995:00:11:51 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+dynip38.efn.org - - [01/Jul/1995:00:11:51 -0400] "GET /images/construct.gif HTTP/1.0" 304 0
+dynip38.efn.org - - [01/Jul/1995:00:11:54 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+200.13.2.37 - - [01/Jul/1995:00:11:55 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+teleman.pr.mcs.net - - [01/Jul/1995:00:11:55 -0400] "GET /cgi-bin/imagemap/countdown?106,212 HTTP/1.0" 302 95
+ppp3_136.bekkoame.or.jp - - [01/Jul/1995:00:11:55 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+teleman.pr.mcs.net - - [01/Jul/1995:00:11:56 -0400] "GET /shuttle/countdown/tour.html HTTP/1.0" 200 4347
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:11:57 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ppp111.cs.mci.com - - [01/Jul/1995:00:11:58 -0400] "GET /shuttle/missions/sts-74/mission-sts-74.html HTTP/1.0" 200 3707
+teleman.pr.mcs.net - - [01/Jul/1995:00:11:58 -0400] "GET /images/KSC-94EC-412-small.gif HTTP/1.0" 200 20484
+ppp111.cs.mci.com - - [01/Jul/1995:00:12:02 -0400] "GET /shuttle/missions/sts-74/sts-74-patch-small.gif HTTP/1.0" 200 4179
+dynip38.efn.org - - [01/Jul/1995:00:12:02 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+204.19.123.36 - - [01/Jul/1995:00:12:03 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:12:06 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+dynip38.efn.org - - [01/Jul/1995:00:12:07 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+leet.cts.com - - [01/Jul/1995:00:12:08 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:12:09 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+205.157.131.144 - - [01/Jul/1995:00:12:10 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.gif HTTP/1.0" 200 29634
+slip1.kias.com - - [01/Jul/1995:00:12:11 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+www-b2.proxy.aol.com - - [01/Jul/1995:00:12:11 -0400] "GET / HTTP/1.0" 200 7074
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:12:12 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+piweba1y.prodigy.com - - [01/Jul/1995:00:12:13 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+www-d3.proxy.aol.com - - [01/Jul/1995:00:12:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.gif HTTP/1.0" 200 25814
+dynip38.efn.org - - [01/Jul/1995:00:12:16 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:17 -0400] "GET /history/apollo/apollo-13/movies/apo13damage.mpg HTTP/1.0" 200 73728
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:12:18 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dynip38.efn.org - - [01/Jul/1995:00:12:19 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+dynip38.efn.org - - [01/Jul/1995:00:12:20 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+teleman.pr.mcs.net - - [01/Jul/1995:00:12:20 -0400] "GET /cgi-bin/imagemap/countdown?321,276 HTTP/1.0" 302 98
+www-a1.proxy.aol.com - - [01/Jul/1995:00:12:20 -0400] "GET /facilities/mlp.html HTTP/1.0" 200 2653
+www-b2.proxy.aol.com - - [01/Jul/1995:00:12:20 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+www-b2.proxy.aol.com - - [01/Jul/1995:00:12:20 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+200.13.2.37 - - [01/Jul/1995:00:12:23 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+200.13.2.37 - - [01/Jul/1995:00:12:23 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ppp-mia-53.shadow.net - - [01/Jul/1995:00:12:24 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0423.jpg HTTP/1.0" 200 84494
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:12:24 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+teleman.pr.mcs.net - - [01/Jul/1995:00:12:26 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:12:26 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:28 -0400] "GET /history/apollo/apollo-13/movies/apo13damage.mpg HTTP/1.0" 200 57344
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:12:28 -0400] "GET /ksc.html HTTP/1.0" 304 0
+www-b2.proxy.aol.com - - [01/Jul/1995:00:12:30 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ppp046.st.rim.or.jp - - [01/Jul/1995:00:12:30 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:31 -0400] "GET /history/apollo/apollo-13/ HTTP/1.0" 200 1732
+piweba1y.prodigy.com - - [01/Jul/1995:00:12:31 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:33 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:33 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+www-d3.proxy.aol.com - - [01/Jul/1995:00:12:34 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:12:36 -0400] "GET /history/history.html HTTP/1.0" 304 0
+205.157.131.144 - - [01/Jul/1995:00:12:37 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.jpg HTTP/1.0" 200 39654
+ppp4.sunrem.com - - [01/Jul/1995:00:12:38 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+www-a1.proxy.aol.com - - [01/Jul/1995:00:12:40 -0400] "GET /images/mlp-logo.gif HTTP/1.0" 200 28426
+pm28.sonic.net - - [01/Jul/1995:00:12:40 -0400] "GET /cgi-bin/imagemap/countdown?103,142 HTTP/1.0" 302 96
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:12:40 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 304 0
+piweba1y.prodigy.com - - [01/Jul/1995:00:12:42 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+burger.letters.com - - [01/Jul/1995:00:12:42 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+burger.letters.com - - [01/Jul/1995:00:12:44 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ppp4.sunrem.com - - [01/Jul/1995:00:12:45 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+dd11-054.compuserve.com - - [01/Jul/1995:00:12:49 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+129.188.154.200 - - [01/Jul/1995:00:12:51 -0400] "GET /shuttle HTTP/1.0" 302 -
+166.79.67.111 - - [01/Jul/1995:00:12:53 -0400] "GET /shuttle/countdown/ HTTP/1.0" 304 0
+dd11-054.compuserve.com - - [01/Jul/1995:00:12:53 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+teleman.pr.mcs.net - - [01/Jul/1995:00:12:54 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 73132
+129.188.154.200 - - [01/Jul/1995:00:12:55 -0400] "GET /shuttle/ HTTP/1.0" 200 957
+waters-gw.starway.net.au - - [01/Jul/1995:00:12:55 -0400] "GET /shuttle/missions/51-l/images/86HC68.GIF HTTP/1.0" 200 116798
+dnet018.sat.texas.net - - [01/Jul/1995:00:12:57 -0400] "GET /history/apollo/apollo-13/videos/ HTTP/1.0" 200 381
+burger.letters.com - - [01/Jul/1995:00:12:57 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 73132
+www-d3.proxy.aol.com - - [01/Jul/1995:00:13:02 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+dnet018.sat.texas.net - - [01/Jul/1995:00:13:04 -0400] "GET /history/apollo/apollo-13/ HTTP/1.0" 200 1732
+205.188.0.173 - - [01/Jul/1995:00:13:05 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:05 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:13:06 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-hatch-hand-group.mpg HTTP/1.0" 200 1081049
+129.188.154.200 - - [01/Jul/1995:00:13:06 -0400] "GET / HTTP/1.0" 200 7074
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:07 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:13:07 -0400] "GET /history/apollo/apollo-12/apollo-12.html HTTP/1.0" 304 0
+166.79.67.111 - - [01/Jul/1995:00:13:09 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+dd11-054.compuserve.com - - [01/Jul/1995:00:13:11 -0400] "GET /shuttle/technology/sts-newsref/sts-lcc.html HTTP/1.0" 200 32252
+uconnvm.uconn.edu - - [01/Jul/1995:00:13:13 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+166.79.67.111 - - [01/Jul/1995:00:13:20 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:20 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:20 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+166.79.67.111 - - [01/Jul/1995:00:13:21 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:13:25 -0400] "GET /history/skylab/skylab.html HTTP/1.0" 200 1687
+204.212.254.71 - - [01/Jul/1995:00:13:25 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:13:26 -0400] "GET /history/skylab/skylab-small.gif HTTP/1.0" 200 9202
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:13:27 -0400] "GET /shuttle/missions/sts-71/sts-71-press-kit.txt HTTP/1.0" 200 78588
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:28 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ppp4.sunrem.com - - [01/Jul/1995:00:13:29 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+204.212.254.71 - - [01/Jul/1995:00:13:30 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+204.212.254.71 - - [01/Jul/1995:00:13:31 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:31 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:32 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:32 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:33 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:33 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:33 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:34 -0400] "GET /images/construct.gif HTTP/1.0" 304 0
+kenmarks-ppp.clark.net - - [01/Jul/1995:00:13:34 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 304 0
+slip1.kias.com - - [01/Jul/1995:00:13:36 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:13:38 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+204.212.254.71 - - [01/Jul/1995:00:13:38 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:13:40 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:42 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:43 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:13:46 -0400] "GET /history/skylab/skylab-3.html HTTP/1.0" 200 1424
+waters-gw.starway.net.au - - [01/Jul/1995:00:13:47 -0400] "GET /shuttle/missions/51-l/movies/ HTTP/1.0" 200 372
+oahu-53.u.aloha.net - - [01/Jul/1995:00:13:48 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:13:48 -0400] "GET /history/skylab/skylab-logo.gif HTTP/1.0" 200 3274
+sfsp86.slip.net - - [01/Jul/1995:00:13:50 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:13:53 -0400] "GET /history/apollo/apollo-11/images/69HC680.GIF HTTP/1.0" 200 149444
+ppp-nyc-3-1.ios.com - - [01/Jul/1995:00:13:53 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-hatch-hand-group.mpg HTTP/1.0" 200 1081049
+teleman.pr.mcs.net - - [01/Jul/1995:00:14:11 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock.mpg HTTP/1.0" 200 122880
+temprano.netheaven.com - - [01/Jul/1995:00:14:17 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:14:17 -0400] "GET /persons/astronauts/i-to-l/lousmaJR.txt HTTP/1.0" 404 -
+waters-gw.starway.net.au - - [01/Jul/1995:00:14:18 -0400] "GET /shuttle/missions/51-l/docs/ HTTP/1.0" 200 368
+temprano.netheaven.com - - [01/Jul/1995:00:14:20 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+temprano.netheaven.com - - [01/Jul/1995:00:14:20 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+pm110.spectra.net - - [01/Jul/1995:00:14:21 -0400] "GET /shuttle/missions/sts-71/sts-71-day-01-highlights.html HTTP/1.0" 200 2722
+129.188.154.200 - - [01/Jul/1995:00:14:21 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:14:22 -0400] "GET /history/skylab/skylab-logo.gif HTTP/1.0" 200 3274
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:14:23 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0893.jpg HTTP/1.0" 200 298302
+ppp4.sunrem.com - - [01/Jul/1995:00:14:24 -0400] "GET /cgi-bin/imagemap/countdown?266,274 HTTP/1.0" 302 85
+temprano.netheaven.com - - [01/Jul/1995:00:14:26 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+sfsp86.slip.net - - [01/Jul/1995:00:14:27 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+sfsp86.slip.net - - [01/Jul/1995:00:14:30 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+boing.dgsys.com - - [01/Jul/1995:00:14:30 -0400] "GET /facts/faq11.html HTTP/1.0" 200 22096
+ppp4.sunrem.com - - [01/Jul/1995:00:14:31 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:14:32 -0400] "GET /persons/astronauts/a-to-d/beanAL.txt HTTP/1.0" 404 -
+waters-gw.starway.net.au - - [01/Jul/1995:00:14:36 -0400] "GET /shuttle/missions/51-l/news/ HTTP/1.0" 200 368
+slip-5.io.com - - [01/Jul/1995:00:14:36 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+ottgate2.bnr.ca - - [01/Jul/1995:00:14:37 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 304 0
+waters-gw.starway.net.au - - [01/Jul/1995:00:15:53 -0400] "GET /htbin/wais.pl?51-L HTTP/1.0" 200 316
+dd11-054.compuserve.com - - [01/Jul/1995:00:15:55 -0400] "GET /shuttle/technology/sts-newsref/sts-lcc.html HTTP/1.0" 200 32252
+204.212.254.71 - - [01/Jul/1995:00:15:58 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+204.227.13.26 - - [01/Jul/1995:00:15:58 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip-5.io.com - - [01/Jul/1995:00:15:59 -0400] "GET /history/apollo/as-201/as-201.html HTTP/1.0" 200 3680
+slip-5.io.com - - [01/Jul/1995:00:16:02 -0400] "GET /history/apollo/as-201/as-201-patch-small.gif HTTP/1.0" 200 16184
+204.227.13.26 - - [01/Jul/1995:00:16:02 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+204.227.13.26 - - [01/Jul/1995:00:16:02 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:03 -0400] "GET /shuttle/missions/sts-74/mission-sts-74.html HTTP/1.0" 200 3707
+ix-sd11-26.ix.netcom.com - - [01/Jul/1995:00:16:03 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:16:04 -0400] "GET /shuttle/missions/sts-68/ksc-srl-image.html HTTP/1.0" 200 1404
+156.151.176.30 - - [01/Jul/1995:00:16:04 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+slip-5.io.com - - [01/Jul/1995:00:16:05 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+204.212.254.71 - - [01/Jul/1995:00:16:05 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+slip-5.io.com - - [01/Jul/1995:00:16:08 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:09 -0400] "GET /shuttle/missions/sts-74/sts-74-patch-small.gif HTTP/1.0" 200 4179
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:09 -0400] "GET /history/apollo/apollo-13/docs/ HTTP/1.0" 200 377
+pm28.sonic.net - - [01/Jul/1995:00:16:10 -0400] "GET /cgi-bin/imagemap/countdown?373,274 HTTP/1.0" 302 68
+detroit.freenet.org - - [01/Jul/1995:00:16:10 -0400] "GET /shuttle/countdown HTTP/1.0" 302 -
+ip155.tmn.com - - [01/Jul/1995:00:16:11 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+204.212.254.71 - - [01/Jul/1995:00:16:12 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+129.188.154.200 - - [01/Jul/1995:00:16:13 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+156.151.176.30 - - [01/Jul/1995:00:16:13 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:16:14 -0400] "GET /facts/facts.html HTTP/1.0" 200 4717
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:15 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 304 0
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:15 -0400] "GET /history/apollo/apollo-13/ HTTP/1.0" 200 1732
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:16:15 -0400] "GET /images/faq.gif HTTP/1.0" 200 263
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:16 -0400] "GET /images/launchmedium.gif HTTP/1.0" 304 0
+detroit.freenet.org - - [01/Jul/1995:00:16:16 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:16 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:17 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:17 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+annex-p2.sci.dixie.edu - - [01/Jul/1995:00:16:20 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:20 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+204.227.13.26 - - [01/Jul/1995:00:16:21 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:21 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+156.151.176.30 - - [01/Jul/1995:00:16:21 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:25 -0400] "GET /history/apollo/apollo-13/videos/ HTTP/1.0" 200 381
+detroit.freenet.org - - [01/Jul/1995:00:16:27 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:29 -0400] "GET / HTTP/1.0" 200 7074
+teleman.pr.mcs.net - - [01/Jul/1995:00:16:30 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock-2.mpg HTTP/1.0" 200 65536
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:16:30 -0400] "GET /history/apollo/apollo-11/images/69HC683.GIF HTTP/1.0" 200 193700
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:16:33 -0400] "GET /facts/faq06.html HTTP/1.0" 200 12686
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:34 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:35 -0400] "GET /shuttle/missions/sts-37/mission-sts-37.html HTTP/1.0" 200 6023
+156.151.176.45 - - [01/Jul/1995:00:16:35 -0400] "GET /ksc.html HTTP/1.0" 304 0
+port2.cia.com - - [01/Jul/1995:00:16:36 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:37 -0400] "GET /shuttle/missions/sts-37/sts-37-patch-small.gif HTTP/1.0" 200 10371
+port2.cia.com - - [01/Jul/1995:00:16:37 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+156.151.176.45 - - [01/Jul/1995:00:16:37 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+156.151.176.45 - - [01/Jul/1995:00:16:37 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 304 0
+156.151.176.45 - - [01/Jul/1995:00:16:37 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 304 0
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:38 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:39 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+port2.cia.com - - [01/Jul/1995:00:16:39 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+port2.cia.com - - [01/Jul/1995:00:16:39 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+156.151.176.45 - - [01/Jul/1995:00:16:40 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:40 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:42 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+shrspine.rapidramp.com - - [01/Jul/1995:00:16:43 -0400] "GET /images/launch-logo.gif HTTP/1.0" 304 0
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:43 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+156.151.176.45 - - [01/Jul/1995:00:16:43 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 304 0
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:44 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:44 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:44 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+www-a1.proxy.aol.com - - [01/Jul/1995:00:16:44 -0400] "GET /images/oldtower.gif HTTP/1.0" 200 173919
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:46 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:46 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:47 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:47 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:47 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:48 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:49 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:50 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:50 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:50 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:16:51 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+129.188.154.200 - - [01/Jul/1995:00:16:51 -0400] "GET /htbin/wais.pl HTTP/1.0" 200 308
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:53 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:16:53 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+ppp111.cs.mci.com - - [01/Jul/1995:00:16:54 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:54 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:54 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:16:56 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:56 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:16:56 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:16:58 -0400] "GET /history/apollo/apollo-goals.txt HTTP/1.0" 200 712
+129.188.154.200 - - [01/Jul/1995:00:17:00 -0400] "GET /shuttle/movies/astronauts.mpg HTTP/1.0" 200 106496
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:17:03 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+129.188.154.200 - - [01/Jul/1995:00:17:03 -0400] "GET /htbin/wais.pl?mpeg HTTP/1.0" 200 1639
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:17:03 -0400] "GET / HTTP/1.0" 200 7074
+shrspine.rapidramp.com - - [01/Jul/1995:00:17:05 -0400] "GET /facts/faq12.html HTTP/1.0" 200 19547
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:05 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:08 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+dialup61.afn.org - - [01/Jul/1995:00:17:09 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+204.227.13.26 - - [01/Jul/1995:00:17:11 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:11 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dialup61.afn.org - - [01/Jul/1995:00:17:12 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ppp5.earthlight.co.nz - - [01/Jul/1995:00:17:13 -0400] "GET /shuttle/missions/sts-70/mission-sts-70.html HTTP/1.0" 200 13469
+204.227.13.26 - - [01/Jul/1995:00:17:13 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+sfsp86.slip.net - - [01/Jul/1995:00:17:16 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+news.ti.com - - [01/Jul/1995:00:17:19 -0400] "GET /shuttle/missions/sts-71/news HTTP/1.0" 302 -
+dialup61.afn.org - - [01/Jul/1995:00:17:19 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:17:19 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:17:20 -0400] "GET /shuttle/missions/sts-71/news/ HTTP/1.0" 200 3442
+dialup61.afn.org - - [01/Jul/1995:00:17:21 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 304 0
+dialup61.afn.org - - [01/Jul/1995:00:17:22 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+204.227.13.26 - - [01/Jul/1995:00:17:22 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+news.ti.com - - [01/Jul/1995:00:17:23 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+news.ti.com - - [01/Jul/1995:00:17:23 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+news.ti.com - - [01/Jul/1995:00:17:23 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+ix-dc10-19.ix.netcom.com - - [01/Jul/1995:00:17:24 -0400] "GET /cgi-bin/imagemap/countdown?375,272 HTTP/1.0" 302 68
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:17:28 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:17:28 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+shrspine.rapidramp.com - - [01/Jul/1995:00:17:28 -0400] "GET /images/faq.gif HTTP/1.0" 200 263
+shrspine.rapidramp.com - - [01/Jul/1995:00:17:29 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:29 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:17:29 -0400] "GET /shuttle/missions/sts-68/images/ksc.gif HTTP/1.0" 200 152676
+detroit.freenet.org - - [01/Jul/1995:00:17:29 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock.mpg HTTP/1.0" 200 946425
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:31 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+129.188.154.200 - - [01/Jul/1995:00:17:32 -0400] "GET /shuttle/countdown/lps/mpeg.html HTTP/1.0" 200 1221
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:33 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+129.188.154.200 - - [01/Jul/1995:00:17:34 -0400] "GET /shuttle/countdown/lps/back.gif HTTP/1.0" 200 1289
+pm28.sonic.net - - [01/Jul/1995:00:17:34 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+dd11-054.compuserve.com - - [01/Jul/1995:00:17:35 -0400] "GET /shuttle/missions/sts-67/sts-67-patch.jpg HTTP/1.0" 200 103193
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:17:36 -0400] "GET /history/apollo/apollo-spacecraft.txt HTTP/1.0" 200 2261
+129.188.154.200 - - [01/Jul/1995:00:17:39 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+leet.cts.com - - [01/Jul/1995:00:17:42 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+129.193.116.41 - - [01/Jul/1995:00:17:46 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+129.188.154.200 - - [01/Jul/1995:00:17:47 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+129.188.154.200 - - [01/Jul/1995:00:17:49 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+129.193.116.41 - - [01/Jul/1995:00:17:51 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+129.193.116.41 - - [01/Jul/1995:00:17:52 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+129.193.116.41 - - [01/Jul/1995:00:17:52 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:17:54 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+news.ti.com - - [01/Jul/1995:00:17:54 -0400] "GET /shuttle/missions/sts-71/sts-71-info.html HTTP/1.0" 200 1440
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:55 -0400] "GET /shuttle/missions/sts-77/mission-sts-77.html HTTP/1.0" 200 3071
+traitor.demon.co.uk - - [01/Jul/1995:00:17:57 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:17:57 -0400] "GET /shuttle/missions/sts-77/sts-77-patch-small.gif HTTP/1.0" 200 4179
+blv-pm0-ip28.halcyon.com - - [01/Jul/1995:00:18:01 -0400] "GET /facts/launch-pass.txt HTTP/1.0" 200 29823
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:18:02 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+sfsp86.slip.net - - [01/Jul/1995:00:18:02 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+204.227.13.26 - - [01/Jul/1995:00:18:03 -0400] "GET /cgi-bin/imagemap/countdown?99,144 HTTP/1.0" 302 96
+204.248.98.63 - - [01/Jul/1995:00:18:05 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip-5.io.com - - [01/Jul/1995:00:18:06 -0400] "GET /history/apollo/as-201/as-201-info.html HTTP/1.0" 200 1395
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:18:08 -0400] "GET /shuttle/missions/sts-76/mission-sts-76.html HTTP/1.0" 200 3109
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:18:08 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:18:10 -0400] "GET /shuttle/missions/sts-76/sts-76-patch-small.gif HTTP/1.0" 200 4179
+204.248.98.63 - - [01/Jul/1995:00:18:10 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+204.248.98.63 - - [01/Jul/1995:00:18:10 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+204.248.98.63 - - [01/Jul/1995:00:18:10 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+slip-5.io.com - - [01/Jul/1995:00:18:13 -0400] "GET /history/apollo/as-201/sounds/ HTTP/1.0" 200 372
+traitor.demon.co.uk - - [01/Jul/1995:00:18:13 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+ppp160.iadfw.net - - [01/Jul/1995:00:18:13 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+traitor.demon.co.uk - - [01/Jul/1995:00:18:13 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+slip-5.io.com - - [01/Jul/1995:00:18:15 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+slip-5.io.com - - [01/Jul/1995:00:18:15 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:18:16 -0400] "GET /software/winvn/wvlarge.gif HTTP/1.0" 200 23416
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:18:20 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+net-1-217.eden.com - - [01/Jul/1995:00:18:20 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+slip-5.io.com - - [01/Jul/1995:00:18:23 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+pm28.sonic.net - - [01/Jul/1995:00:18:23 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0613.gif HTTP/1.0" 200 45970
+slip-5.io.com - - [01/Jul/1995:00:18:25 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+slip-5.io.com - - [01/Jul/1995:00:18:25 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+www-a1.proxy.aol.com - - [01/Jul/1995:00:18:27 -0400] "GET /facilities/vab.html HTTP/1.0" 200 4045
+teleman.pr.mcs.net - - [01/Jul/1995:00:18:28 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock-2.mpg HTTP/1.0" 200 139264
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:18:28 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ppp160.iadfw.net - - [01/Jul/1995:00:18:28 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:18:29 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:18:30 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+whlane.cts.com - - [01/Jul/1995:00:18:31 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+whlane.cts.com - - [01/Jul/1995:00:18:33 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+www-a1.proxy.aol.com - - [01/Jul/1995:00:18:33 -0400] "GET /images/vab-small.gif HTTP/1.0" 200 35709
+slip-5.io.com - - [01/Jul/1995:00:18:33 -0400] "GET /history/apollo/as-201/sounds/ HTTP/1.0" 200 372
+ottgate2.bnr.ca - - [01/Jul/1995:00:18:34 -0400] "GET /shuttle/technology/images/et_1-small.gif HTTP/1.0" 200 35624
+ppp5.earthlight.co.nz - - [01/Jul/1995:00:18:35 -0400] "GET /shuttle/missions/sts-70/sts-70-info.html HTTP/1.0" 200 1429
+ppp24.swcp.com - - [01/Jul/1995:00:18:35 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ottgate2.bnr.ca - - [01/Jul/1995:00:18:37 -0400] "GET /shuttle/technology/images/et-lox_1-small.gif HTTP/1.0" 200 36098
+ppp24.swcp.com - - [01/Jul/1995:00:18:39 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ppp24.swcp.com - - [01/Jul/1995:00:18:39 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ppp24.swcp.com - - [01/Jul/1995:00:18:39 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:18:39 -0400] "GET /pub/winvn/readme.txt HTTP/1.0" 404 -
+whlane.cts.com - - [01/Jul/1995:00:18:41 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+whlane.cts.com - - [01/Jul/1995:00:18:41 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+slip-5.io.com - - [01/Jul/1995:00:18:41 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+204.227.13.26 - - [01/Jul/1995:00:18:42 -0400] "GET /cgi-bin/imagemap/countdown?98,108 HTTP/1.0" 302 111
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:18:42 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:18:43 -0400] "GET /shuttle/missions/sts-68/images/ksc-upclose.gif HTTP/1.0" 200 86984
+burger.letters.com - - [01/Jul/1995:00:18:43 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+ppp160.iadfw.net - - [01/Jul/1995:00:18:44 -0400] "GET /htbin/wais.pl HTTP/1.0" 200 308
+burger.letters.com - - [01/Jul/1995:00:18:45 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:18:45 -0400] "GET /pub/winvn/readme.txt HTTP/1.0" 404 -
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:18:45 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ppp5.earthlight.co.nz - - [01/Jul/1995:00:18:46 -0400] "GET /shuttle/missions/sts-70/images/ HTTP/1.0" 200 966
+cu-dialup-1005.cit.cornell.edu - - [01/Jul/1995:00:18:49 -0400] "GET /pub/winvn/release.txt HTTP/1.0" 404 -
+ppp160.iadfw.net - - [01/Jul/1995:00:18:52 -0400] "GET /htbin/wais.pl?sarex HTTP/1.0" 200 7083
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:18:52 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+temprano.netheaven.com - - [01/Jul/1995:00:18:56 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+burger.letters.com - - [01/Jul/1995:00:18:57 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 70412
+mimas.execpc.com - - [01/Jul/1995:00:18:59 -0400] "GET /shuttle/missions/technology/sts-newsref/stsref-toc.html HTTP/1.0" 404 -
+slip-5.io.com - - [01/Jul/1995:00:18:59 -0400] "GET /history/apollo/as-201/sounds/ HTTP/1.0" 200 372
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:18:59 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:19:01 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:01 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+traitor.demon.co.uk - - [01/Jul/1995:00:19:02 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 304 0
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:19:02 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+whlane.cts.com - - [01/Jul/1995:00:19:02 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:02 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:03 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:03 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:19:04 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+dialup61.afn.org - - [01/Jul/1995:00:19:04 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 304 0
+whlane.cts.com - - [01/Jul/1995:00:19:04 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+slip-5.io.com - - [01/Jul/1995:00:19:05 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+news.ti.com - - [01/Jul/1995:00:19:05 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+dialup61.afn.org - - [01/Jul/1995:00:19:06 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 304 0
+remote27.compusmart.ab.ca - - [01/Jul/1995:00:19:06 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:19:08 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+slip-5.io.com - - [01/Jul/1995:00:19:10 -0400] "GET /history/apollo/as-201/movies/ HTTP/1.0" 200 372
+traitor.demon.co.uk - - [01/Jul/1995:00:19:11 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 304 0
+129.193.116.41 - - [01/Jul/1995:00:19:11 -0400] "GET /cgi-bin/imagemap/countdown?370,276 HTTP/1.0" 302 68
+traitor.demon.co.uk - - [01/Jul/1995:00:19:12 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 304 0
+janice.cc.wwu.edu - - [01/Jul/1995:00:19:12 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:19:13 -0400] "GET /history/apollo/apollo-11/images/69HC684.GIF HTTP/1.0" 200 101647
+slip-5.io.com - - [01/Jul/1995:00:19:13 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:13 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:19:14 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+traitor.demon.co.uk - - [01/Jul/1995:00:19:14 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+whlane.cts.com - - [01/Jul/1995:00:19:15 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+midcom.com - - [01/Jul/1995:00:19:16 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:19:17 -0400] "GET /shuttle/missions/sts-71/news HTTP/1.0" 302 -
+acs4.acs.ucalgary.ca - - [01/Jul/1995:00:19:17 -0400] "GET /shuttle/missions/sts-71/news/ HTTP/1.0" 200 3442
+net-1-217.eden.com - - [01/Jul/1995:00:19:18 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0876.jpg HTTP/1.0" 200 87766
+slip-5.io.com - - [01/Jul/1995:00:19:18 -0400] "GET /history/apollo/as-201/images/ HTTP/1.0" 200 678
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:22 -0400] "GET /shuttle/missions/sts-1/mission-sts-1.html HTTP/1.0" 200 8149
+traitor.demon.co.uk - - [01/Jul/1995:00:19:23 -0400] "GET /history/apollo/apollo-13/docs/ HTTP/1.0" 200 377
+traitor.demon.co.uk - - [01/Jul/1995:00:19:24 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+traitor.demon.co.uk - - [01/Jul/1995:00:19:24 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+janice.cc.wwu.edu - - [01/Jul/1995:00:19:25 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 48591
+ottgate2.bnr.ca - - [01/Jul/1995:00:19:26 -0400] "GET /shuttle/technology/images/et-intertank_1-small.gif HTTP/1.0" 200 79791
+ppp160.iadfw.net - - [01/Jul/1995:00:19:27 -0400] "GET /shuttle/missions/sts-71/sts-71-press-kit.txt HTTP/1.0" 200 78588
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:30 -0400] "GET /history/history.html HTTP/1.0" 200 1502
+204.227.13.26 - - [01/Jul/1995:00:19:30 -0400] "GET /shuttle/missions/sts-71/sts-71-day-04-highlights.html HTTP/1.0" 200 5544
+cruzio.com - - [01/Jul/1995:00:19:31 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:31 -0400] "GET /history/apollo/images/apollo-small.gif HTTP/1.0" 200 9630
+whlane.cts.com - - [01/Jul/1995:00:19:32 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:32 -0400] "GET /shuttle/resources/orbiters/columbia.html HTTP/1.0" 200 6922
+news.ti.com - - [01/Jul/1995:00:19:33 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+cruzio.com - - [01/Jul/1995:00:19:35 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 47296
+204.248.98.63 - - [01/Jul/1995:00:19:36 -0400] "GET /cgi-bin/imagemap/countdown?383,277 HTTP/1.0" 302 68
+traitor.demon.co.uk - - [01/Jul/1995:00:19:37 -0400] "GET /history/apollo/apollo-13/movies/ HTTP/1.0" 200 945
+ppp24.swcp.com - - [01/Jul/1995:00:19:37 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+slip-5.io.com - - [01/Jul/1995:00:19:38 -0400] "GET /history/apollo/as-201/news/ HTTP/1.0" 200 368
+whlane.cts.com - - [01/Jul/1995:00:19:38 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+traitor.demon.co.uk - - [01/Jul/1995:00:19:38 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:40 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:40 -0400] "GET /cgi-bin/imagemap/countdown?93,109 HTTP/1.0" 302 111
+dal31.pic.net - - [01/Jul/1995:00:19:41 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:41 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:41 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+slip-5.io.com - - [01/Jul/1995:00:19:42 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:42 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+p43.infinet.com - - [01/Jul/1995:00:19:42 -0400] "GET /images/shuttle-patch-small.gif HTTP/1.0" 200 4179
+ix-roc1-07.ix.netcom.com - - [01/Jul/1995:00:19:42 -0400] "GET /shuttle/technology/images/mission_profile_2-small.gif HTTP/1.0" 200 35540
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:43 -0400] "GET /shuttle/resources/orbiters/columbia-logo.gif HTTP/1.0" 200 11417
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:43 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:43 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+dal31.pic.net - - [01/Jul/1995:00:19:45 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ppp6.cpbx.net - - [01/Jul/1995:00:19:45 -0400] "HEAD /software/winvn/winvn.html HTTP/1.0" 200 0
+slip-5.io.com - - [01/Jul/1995:00:19:46 -0400] "GET /history/apollo/as-201/news/ HTTP/1.0" 200 368
+traitor.demon.co.uk - - [01/Jul/1995:00:19:50 -0400] "GET /history/apollo/apollo-13/sounds/ HTTP/1.0" 200 1157
+traitor.demon.co.uk - - [01/Jul/1995:00:19:51 -0400] "GET /icons/sound.xbm HTTP/1.0" 200 530
+sfsp86.slip.net - - [01/Jul/1995:00:19:53 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0423.gif HTTP/1.0" 200 64939
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:19:53 -0400] "GET /history/apollo/apollo-11/movies/ HTTP/1.0" 200 381
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:54 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:19:54 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+teleman.pr.mcs.net - - [01/Jul/1995:00:19:56 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+temprano.netheaven.com - - [01/Jul/1995:00:19:57 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+magicall.dacom.co.kr - - [01/Jul/1995:00:19:58 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+slip-5.io.com - - [01/Jul/1995:00:19:58 -0400] "GET /history/apollo/as-201/sounds/ HTTP/1.0" 200 372
+205.147.133.13 - - [01/Jul/1995:00:19:58 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:19:58 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+leet.cts.com - - [01/Jul/1995:00:20:01 -0400] "GET /shuttle/missions/sts-71/images/index71.gif HTTP/1.0" 200 235862
+202.70.0.6 - - [01/Jul/1995:00:20:03 -0400] "GET /shuttle/missions/sts-63/sts-63-info.html HTTP/1.0" 200 1431
+slip-5.io.com - - [01/Jul/1995:00:20:04 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+magicall.dacom.co.kr - - [01/Jul/1995:00:20:05 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+202.70.0.6 - - [01/Jul/1995:00:20:05 -0400] "GET /shuttle/missions/sts-63/sts-63-patch-small.gif HTTP/1.0" 200 16102
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:20:06 -0400] "GET /history/apollo/apollo-11/sounds/ HTTP/1.0" 200 653
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:20:06 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-roc1-07.ix.netcom.com - - [01/Jul/1995:00:20:07 -0400] "GET /shuttle/technology/images/tal_abort_2-small.gif HTTP/1.0" 200 10099
+205.147.133.13 - - [01/Jul/1995:00:20:07 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+teleman.pr.mcs.net - - [01/Jul/1995:00:20:09 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+ppp5.earthlight.co.nz - - [01/Jul/1995:00:20:11 -0400] "GET /htbin/wais.pl?SAREX-II HTTP/1.0" 200 7124
+dal31.pic.net - - [01/Jul/1995:00:20:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 71311
+202.70.0.6 - - [01/Jul/1995:00:20:12 -0400] "GET /shuttle/missions/sts-63/sounds/ HTTP/1.0" 200 378
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:12 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 57344
+magicall.dacom.co.kr - - [01/Jul/1995:00:20:13 -0400] "GET /shuttle/missions/sts-73/mission-sts-73.html HTTP/1.0" 200 4085
+202.70.0.6 - - [01/Jul/1995:00:20:14 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+202.70.0.6 - - [01/Jul/1995:00:20:14 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+slip-5.io.com - - [01/Jul/1995:00:20:14 -0400] "GET /history/apollo/as-201/news/ HTTP/1.0" 200 368
+whlane.cts.com - - [01/Jul/1995:00:20:14 -0400] "GET /cgi-bin/imagemap/countdown?97,146 HTTP/1.0" 302 96
+traitor.demon.co.uk - - [01/Jul/1995:00:20:15 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:16 -0400] "GET / HTTP/1.0" 200 7074
+news.ti.com - - [01/Jul/1995:00:20:16 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+slip-5.io.com - - [01/Jul/1995:00:20:17 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+202.70.0.6 - - [01/Jul/1995:00:20:18 -0400] "GET /shuttle/missions/sts-63/ HTTP/1.0" 200 2032
+ppp24.swcp.com - - [01/Jul/1995:00:20:20 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0891.jpg HTTP/1.0" 200 80678
+202.70.0.6 - - [01/Jul/1995:00:20:20 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+202.70.0.6 - - [01/Jul/1995:00:20:20 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+magicall.dacom.co.kr - - [01/Jul/1995:00:20:21 -0400] "GET /shuttle/missions/sts-73/sts-73-patch-small.gif HTTP/1.0" 200 4179
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:20:22 -0400] "GET /htbin/cdt_main.pl HTTP/1.0" 200 3214
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:22 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+net-1-217.eden.com - - [01/Jul/1995:00:20:24 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.jpg HTTP/1.0" 200 45966
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:24 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+p43.infinet.com - - [01/Jul/1995:00:20:26 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 84907
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:20:26 -0400] "GET / HTTP/1.0" 200 7074
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:20:27 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+134.20.175.12 - - [01/Jul/1995:00:20:27 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:20:27 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:29 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+slip-5.io.com - - [01/Jul/1995:00:20:29 -0400] "GET /history/apollo/as-201/docs/ HTTP/1.0" 200 368
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:29 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+magicall.dacom.co.kr - - [01/Jul/1995:00:20:29 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+205.147.133.13 - - [01/Jul/1995:00:20:29 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+134.20.175.12 - - [01/Jul/1995:00:20:30 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+134.20.175.12 - - [01/Jul/1995:00:20:30 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:31 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+134.20.175.12 - - [01/Jul/1995:00:20:31 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+205.147.133.13 - - [01/Jul/1995:00:20:32 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+www-a1.proxy.aol.com - - [01/Jul/1995:00:20:32 -0400] "GET /facilities/crawler.html HTTP/1.0" 200 3601
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:33 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+magicall.dacom.co.kr - - [01/Jul/1995:00:20:34 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:34 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:20:35 -0400] "GET /shuttle/missions/sts-70/mission-sts-70.html HTTP/1.0" 200 13469
+www-a1.proxy.aol.com - - [01/Jul/1995:00:20:36 -0400] "GET /images/crawler-logo.gif HTTP/1.0" 200 14108
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:20:38 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+slip-5.io.com - - [01/Jul/1995:00:20:38 -0400] "GET /history/apollo/as-201/ HTTP/1.0" 200 1699
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:20:39 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 84907
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:39 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 57344
+slip-5.io.com - - [01/Jul/1995:00:20:41 -0400] "GET /history/apollo/ HTTP/1.0" 200 6245
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:41 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+dnet018.sat.texas.net - - [01/Jul/1995:00:20:43 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+ottgate2.bnr.ca - - [01/Jul/1995:00:20:44 -0400] "GET /shuttle/technology/images/et_1.jpg HTTP/1.0" 200 144114
+202.70.0.6 - - [01/Jul/1995:00:20:45 -0400] "GET /shuttle/missions/sts-63/movies/ HTTP/1.0" 200 378
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:45 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+slip-5.io.com - - [01/Jul/1995:00:20:46 -0400] "GET /history/apollo/apollo-13/ HTTP/1.0" 200 1732
+h-shining.norfolk.infi.net - - [01/Jul/1995:00:20:50 -0400] "GET /shuttle/missions/sts-70/sts-70-patch-small.gif HTTP/1.0" 200 5026
+p43.infinet.com - - [01/Jul/1995:00:20:51 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+leet.cts.com - - [01/Jul/1995:00:20:51 -0400] "GET /shuttle/missions/sts-71/images/index71.gif HTTP/1.0" 304 0
+p43.infinet.com - - [01/Jul/1995:00:20:52 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+slip-5.io.com - - [01/Jul/1995:00:20:53 -0400] "GET /history/apollo/apollo-13/sounds/ HTTP/1.0" 200 1157
+news.ti.com - - [01/Jul/1995:00:20:54 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+slip-5.io.com - - [01/Jul/1995:00:20:55 -0400] "GET /icons/sound.xbm HTTP/1.0" 200 530
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:20:55 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+slip-28-14.ots.utexas.edu - - [01/Jul/1995:00:20:55 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 57344
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:20:55 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dal31.pic.net - - [01/Jul/1995:00:20:56 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 47592
+midcom.com - - [01/Jul/1995:00:20:56 -0400] "GET /history/apollo/apollo-13/sounds/ HTTP/1.0" 200 1157
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:20:57 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+midcom.com - - [01/Jul/1995:00:20:57 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+midcom.com - - [01/Jul/1995:00:20:58 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+midcom.com - - [01/Jul/1995:00:20:58 -0400] "GET /icons/sound.xbm HTTP/1.0" 200 530
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:21:01 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:21:03 -0400] "GET /images/shuttle-patch-small.gif HTTP/1.0" 200 4179
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:21:03 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+ppp24.swcp.com - - [01/Jul/1995:00:21:05 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0913.jpg HTTP/1.0" 200 25439
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:21:06 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+slip-5.io.com - - [01/Jul/1995:00:21:06 -0400] "GET /history/apollo/apollo-13/sounds/a13_003.wav HTTP/1.0" 200 6685
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:21:06 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:21:08 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+annex12-36.dial.umd.edu - - [01/Jul/1995:00:21:10 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+134.20.175.12 - - [01/Jul/1995:00:21:11 -0400] "GET /cgi-bin/imagemap/countdown?110,112 HTTP/1.0" 302 111
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:21:11 -0400] "GET /shuttle/technology/sts-newsref/spacelab.html HTTP/1.0" 200 104916
+134.20.175.12 - - [01/Jul/1995:00:21:12 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:21:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.gif HTTP/1.0" 200 31242
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:21:14 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+134.20.175.12 - - [01/Jul/1995:00:21:15 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:21:16 -0400] "GET / HTTP/1.0" 200 7074
+134.20.175.12 - - [01/Jul/1995:00:21:21 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:21:22 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:21:23 -0400] "GET /shuttle/missions/sts-68/sts-68-patch-small.gif HTTP/1.0" 200 17459
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:21:24 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:21:26 -0400] "GET /shuttle/missions/sts-68/mission-sts-68.html HTTP/1.0" 200 49087
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:21:33 -0400] "GET /msfc/astro_home.html HTTP/1.0" 200 2938
+ana0013.deltanet.com - - [01/Jul/1995:00:21:33 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+ana0013.deltanet.com - - [01/Jul/1995:00:21:36 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+ana0013.deltanet.com - - [01/Jul/1995:00:21:37 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+ana0013.deltanet.com - - [01/Jul/1995:00:21:37 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+ana0013.deltanet.com - - [01/Jul/1995:00:21:38 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+204.97.234.46 - - [01/Jul/1995:00:21:38 -0400] "GET / HTTP/1.0" 200 7074
+midcom.com - - [01/Jul/1995:00:21:39 -0400] "GET /history/apollo/apollo-13/sounds/a13_001.wav HTTP/1.0" 200 73728
+ana0013.deltanet.com - - [01/Jul/1995:00:21:40 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+picard.microsys.net - - [01/Jul/1995:00:21:40 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+netcom6.netcom.com - - [01/Jul/1995:00:21:41 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+netcom6.netcom.com - - [01/Jul/1995:00:21:43 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+netcom6.netcom.com - - [01/Jul/1995:00:21:43 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+netcom6.netcom.com - - [01/Jul/1995:00:21:43 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+204.97.234.46 - - [01/Jul/1995:00:21:43 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ana0013.deltanet.com - - [01/Jul/1995:00:21:44 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+annex12-36.dial.umd.edu - - [01/Jul/1995:00:21:45 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0589.jpg HTTP/1.0" 200 64427
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:21:47 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+204.97.234.46 - - [01/Jul/1995:00:21:47 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+net-1-217.eden.com - - [01/Jul/1995:00:21:47 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0912.jpg HTTP/1.0" 200 66202
+202.36.46.52 - - [01/Jul/1995:00:21:48 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+dal31.pic.net - - [01/Jul/1995:00:21:48 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock-2.mpg HTTP/1.0" 200 49152
+204.97.234.46 - - [01/Jul/1995:00:21:49 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:21:49 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+netcom6.netcom.com - - [01/Jul/1995:00:21:50 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+204.97.234.46 - - [01/Jul/1995:00:21:50 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:21:50 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ana0013.deltanet.com - - [01/Jul/1995:00:21:51 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ana0013.deltanet.com - - [01/Jul/1995:00:21:51 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+204.97.234.46 - - [01/Jul/1995:00:21:51 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:21:51 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ppp24.swcp.com - - [01/Jul/1995:00:21:52 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.jpg HTTP/1.0" 200 34029
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:21:52 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+netcom6.netcom.com - - [01/Jul/1995:00:21:53 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:21:54 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:21:54 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+dnet018.sat.texas.net - - [01/Jul/1995:00:21:54 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+134.20.175.12 - - [01/Jul/1995:00:21:54 -0400] "GET /shuttle/countdown/ HTTP/1.0" 304 0
+netcom6.netcom.com - - [01/Jul/1995:00:21:55 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+134.20.175.12 - - [01/Jul/1995:00:21:57 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+dal31.pic.net - - [01/Jul/1995:00:21:57 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:21:58 -0400] "GET /shuttle/technology/sts-newsref/sts_asm.html HTTP/1.0" 200 71656
+netcom6.netcom.com - - [01/Jul/1995:00:21:58 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:21:59 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+dal31.pic.net - - [01/Jul/1995:00:21:59 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+129.59.205.2 - - [01/Jul/1995:00:22:02 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+midcom.com - - [01/Jul/1995:00:22:03 -0400] "GET /history/apollo/apollo-13/sounds/a13_001.wav HTTP/1.0" 200 81920
+129.59.205.2 - - [01/Jul/1995:00:22:04 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dal31.pic.net - - [01/Jul/1995:00:22:04 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+dal31.pic.net - - [01/Jul/1995:00:22:04 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+129.59.205.2 - - [01/Jul/1995:00:22:04 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:22:04 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+129.59.205.2 - - [01/Jul/1995:00:22:04 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+netcom6.netcom.com - - [01/Jul/1995:00:22:07 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:22:08 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:22:09 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+sam-slip-l6.neosoft.com - - [01/Jul/1995:00:22:09 -0400] "GET /shuttle/countdown HTTP/1.0" 302 -
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:22:09 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:22:09 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:22:10 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:22:10 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+sam-slip-l6.neosoft.com - - [01/Jul/1995:00:22:10 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+sam-slip-l6.neosoft.com - - [01/Jul/1995:00:22:13 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+sam-slip-l6.neosoft.com - - [01/Jul/1995:00:22:13 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+sam-slip-l6.neosoft.com - - [01/Jul/1995:00:22:14 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:22:16 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+www-a1.proxy.aol.com - - [01/Jul/1995:00:22:19 -0400] "GET /images/rollout.gif HTTP/1.0" 200 258839
+129.59.205.2 - - [01/Jul/1995:00:22:19 -0400] "GET /cgi-bin/imagemap/countdown?89,176 HTTP/1.0" 302 110
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:22:20 -0400] "GET /msfc/astro_home3.gif HTTP/1.0" 200 106535
+129.59.205.2 - - [01/Jul/1995:00:22:20 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+131.128.2.155 - - [01/Jul/1995:00:22:21 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 49152
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:22:22 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:22:22 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0912.gif HTTP/1.0" 200 48305
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:22:23 -0400] "GET /shuttle/technology/images/srb_mod_compare_1-small.gif HTTP/1.0" 200 36902
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:22:25 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:22:25 -0400] "GET /msfc/onboard/onboard.html HTTP/1.0" 200 1301
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:22:27 -0400] "GET /msfc/onboard/redball.gif HTTP/1.0" 200 326
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:22:27 -0400] "GET /msfc/onboard/colorbar.gif HTTP/1.0" 200 796
+midcom.com - - [01/Jul/1995:00:22:27 -0400] "GET /history/apollo/apollo-13/sounds/a13_001.wav HTTP/1.0" 200 98304
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:22:28 -0400] "GET /images/shuttle-patch-logo.gif HTTP/1.0" 200 891
+202.36.46.52 - - [01/Jul/1995:00:22:29 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-dfw11-01.ix.netcom.com - - [01/Jul/1995:00:22:29 -0400] "GET /shuttle/missions/100th.html HTTP/1.0" 200 32303
+chi067.wwa.com - - [01/Jul/1995:00:22:31 -0400] "GET /images HTTP/1.0" 302 -
+129.59.205.2 - - [01/Jul/1995:00:22:32 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0904.jpg HTTP/1.0" 200 55569
+chi067.wwa.com - - [01/Jul/1995:00:22:33 -0400] "GET /images/ HTTP/1.0" 200 17688
+alyssa.prodigy.com - - [01/Jul/1995:00:22:33 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ip149.santa-ana.ca.interramp.com - - [01/Jul/1995:00:22:33 -0400] "GET /msfc/onboard/come_aboard.gif HTTP/1.0" 200 48682
+slip-5.io.com - - [01/Jul/1995:00:22:34 -0400] "GET /history/apollo/apollo-13/sounds/a13_002.wav HTTP/1.0" 200 127045
+dnet018.sat.texas.net - - [01/Jul/1995:00:22:34 -0400] "GET /history/apollo/apollo-13/images/ HTTP/1.0" 200 1851
+midcom.com - - [01/Jul/1995:00:22:37 -0400] "GET /history/apollo/apollo-13/sounds/a13_004.wav HTTP/1.0" 200 66764
+picard.microsys.net - - [01/Jul/1995:00:22:39 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0443.gif HTTP/1.0" 200 64910
+dnet018.sat.texas.net - - [01/Jul/1995:00:22:42 -0400] "GET /history/apollo/apollo-13/ HTTP/1.0" 200 1732
+pipe6.nyc.pipeline.com - - [01/Jul/1995:00:22:43 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock.mpg" 200 946425
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:22:43 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+net-1-217.eden.com - - [01/Jul/1995:00:22:45 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.jpg HTTP/1.0" 200 34029
+midcom.com - - [01/Jul/1995:00:22:47 -0400] "GET /history/apollo/apollo-13/sounds/a13_005.wav HTTP/1.0" 200 81920
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:22:52 -0400] "GET /history/apollo/flight-summary.txt HTTP/1.0" 200 5086
+palona1.cns.hp.com - - [01/Jul/1995:00:22:52 -0400] "GET / HTTP/1.0" 200 7074
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:22:52 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+palona1.cns.hp.com - - [01/Jul/1995:00:22:57 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 304 0
+crl4.crl.com - - [01/Jul/1995:00:22:58 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:22:59 -0400] "GET /history/apollo/apollo-11/sounds/A01108AA.WAV HTTP/1.0" 200 218390
+ppp24.swcp.com - - [01/Jul/1995:00:22:59 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+manx.ccit.arizona.edu - - [01/Jul/1995:00:23:00 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+seds.lpl.arizona.edu - - [01/Jul/1995:00:23:00 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+manx.ccit.arizona.edu - - [01/Jul/1995:00:23:01 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+manx.ccit.arizona.edu - - [01/Jul/1995:00:23:01 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:23:01 -0400] "GET /cgi-bin/imagemap/countdown?376,269 HTTP/1.0" 302 68
+manx.ccit.arizona.edu - - [01/Jul/1995:00:23:01 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+picard.microsys.net - - [01/Jul/1995:00:23:02 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+palona1.cns.hp.com - - [01/Jul/1995:00:23:04 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+149.171.160.183 - - [01/Jul/1995:00:23:05 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+149.171.160.183 - - [01/Jul/1995:00:23:07 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+202.36.46.52 - - [01/Jul/1995:00:23:07 -0400] "GET /shuttle/missions/sts-70/mission-sts-70.html HTTP/1.0" 200 13469
+sartre.execpc.com - - [01/Jul/1995:00:23:07 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:23:08 -0400] "GET /shuttle/technology/images/srb_mod_compare_3-small.gif HTTP/1.0" 200 55666
+manx.ccit.arizona.edu - - [01/Jul/1995:00:23:08 -0400] "GET /cgi-bin/imagemap/countdown?103,144 HTTP/1.0" 302 96
+166.79.67.111 - - [01/Jul/1995:00:23:09 -0400] "GET /shuttle/countdown/ HTTP/1.0" 304 0
+palona1.cns.hp.com - - [01/Jul/1995:00:23:09 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 304 0
+sartre.execpc.com - - [01/Jul/1995:00:23:09 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+149.171.160.183 - - [01/Jul/1995:00:23:10 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+149.171.160.183 - - [01/Jul/1995:00:23:10 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+sartre.execpc.com - - [01/Jul/1995:00:23:10 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+rlnport2.cin.dca.com - - [01/Jul/1995:00:23:11 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+sartre.execpc.com - - [01/Jul/1995:00:23:11 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+166.79.67.111 - - [01/Jul/1995:00:23:13 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:23:14 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+rlnport2.cin.dca.com - - [01/Jul/1995:00:23:15 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+palona1.cns.hp.com - - [01/Jul/1995:00:23:19 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 304 0
+rlnport2.cin.dca.com - - [01/Jul/1995:00:23:20 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:23:20 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0913.gif HTTP/1.0" 200 21957
+rlnport2.cin.dca.com - - [01/Jul/1995:00:23:20 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+dnet018.sat.texas.net - - [01/Jul/1995:00:23:21 -0400] "GET /history/apollo/apollo-13/images/ HTTP/1.0" 200 1851
+dialup61.afn.org - - [01/Jul/1995:00:23:21 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-hand-shake.mpg HTTP/1.0" 200 372172
+palona1.cns.hp.com - - [01/Jul/1995:00:23:23 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:24 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:23:26 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+palona1.cns.hp.com - - [01/Jul/1995:00:23:27 -0400] "GET /facts/facts.html HTTP/1.0" 200 4717
+chi067.wwa.com - - [01/Jul/1995:00:23:27 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+palona1.cns.hp.com - - [01/Jul/1995:00:23:29 -0400] "GET /images/faq.gif HTTP/1.0" 200 263
+palona1.cns.hp.com - - [01/Jul/1995:00:23:31 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:23:31 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:23:31 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+chi067.wwa.com - - [01/Jul/1995:00:23:31 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+net-1-217.eden.com - - [01/Jul/1995:00:23:32 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+chi067.wwa.com - - [01/Jul/1995:00:23:33 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:34 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:23:34 -0400] "GET /cgi-bin/imagemap/countdown?104,169 HTTP/1.0" 302 110
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:23:35 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+chi067.wwa.com - - [01/Jul/1995:00:23:35 -0400] "GET /icons/unknown.xbm HTTP/1.0" 200 515
+www-b5.proxy.aol.com - - [01/Jul/1995:00:23:37 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:23:39 -0400] "GET /shuttle/technology/images/srb_mod_compare_6-small.gif HTTP/1.0" 200 28219
+alyssa.prodigy.com - - [01/Jul/1995:00:23:42 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+149.171.160.183 - - [01/Jul/1995:00:23:43 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+166.79.67.111 - - [01/Jul/1995:00:23:43 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 304 0
+annex12-36.dial.umd.edu - - [01/Jul/1995:00:23:44 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.jpg HTTP/1.0" 200 45966
+149.171.160.183 - - [01/Jul/1995:00:23:46 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:48 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:23:52 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:23:53 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:54 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:57 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+ip155.tmn.com - - [01/Jul/1995:00:23:57 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival.mpg HTTP/1.0" 200 516688
+indy2.indy.net - - [01/Jul/1995:00:23:57 -0400] "GET / HTTP/1.0" 200 7074
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:23:59 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+indy2.indy.net - - [01/Jul/1995:00:24:01 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:24:01 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+palona1.cns.hp.com - - [01/Jul/1995:00:24:04 -0400] "GET /facts/about_ksc.html HTTP/1.0" 200 3977
+chi067.wwa.com - - [01/Jul/1995:00:24:04 -0400] "GET /images/kscmap-tiny.gif HTTP/1.0" 200 2537
+palona1.cns.hp.com - - [01/Jul/1995:00:24:05 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+ix-tam1-26.ix.netcom.com - - [01/Jul/1995:00:24:05 -0400] "GET /images/shuttle-patch-logo.gif HTTP/1.0" 200 891
+rlnport2.cin.dca.com - - [01/Jul/1995:00:24:05 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+palona1.cns.hp.com - - [01/Jul/1995:00:24:06 -0400] "GET /images/launchpalms-small.gif HTTP/1.0" 200 11473
+indy2.indy.net - - [01/Jul/1995:00:24:07 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+166.79.67.111 - - [01/Jul/1995:00:24:08 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+indy2.indy.net - - [01/Jul/1995:00:24:09 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+indy2.indy.net - - [01/Jul/1995:00:24:10 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+indy2.indy.net - - [01/Jul/1995:00:24:11 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+dnet018.sat.texas.net - - [01/Jul/1995:00:24:13 -0400] "GET /history/apollo/apollo-13/images/70HC314.GIF HTTP/1.0" 200 101267
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:24:13 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.gif HTTP/1.0" 200 29634
+net-1-217.eden.com - - [01/Jul/1995:00:24:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.jpg HTTP/1.0" 200 52491
+149.171.160.182 - - [01/Jul/1995:00:24:14 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:24:14 -0400] "GET /facilities/tour.html HTTP/1.0" 200 3723
+crl4.crl.com - - [01/Jul/1995:00:24:16 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.gif HTTP/1.0" 200 29634
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:24:17 -0400] "GET /images/kscmap-small.gif HTTP/1.0" 200 39017
+149.171.160.182 - - [01/Jul/1995:00:24:18 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+149.171.160.182 - - [01/Jul/1995:00:24:19 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+149.171.160.182 - - [01/Jul/1995:00:24:19 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:24:19 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0912.jpg HTTP/1.0" 200 66202
+ppp24.swcp.com - - [01/Jul/1995:00:24:22 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.gif HTTP/1.0" 200 31242
+199.2.253.2 - - [01/Jul/1995:00:24:22 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+rlnport2.cin.dca.com - - [01/Jul/1995:00:24:23 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+teleman.pr.mcs.net - - [01/Jul/1995:00:24:23 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:24:25 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:24:27 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:24:30 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:24:31 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+202.36.46.52 - - [01/Jul/1995:00:24:33 -0400] "GET /whats-new.html HTTP/1.0" 200 17314
+teleman.pr.mcs.net - - [01/Jul/1995:00:24:33 -0400] "GET /history/apollo/apollo-13/sounds/ HTTP/1.0" 200 1157
+teleman.pr.mcs.net - - [01/Jul/1995:00:24:34 -0400] "GET /icons/sound.xbm HTTP/1.0" 200 530
+teleman.pr.mcs.net - - [01/Jul/1995:00:24:34 -0400] "GET /icons/blank.xbm HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:24:34 -0400] "GET /icons/menu.xbm HTTP/1.0" 304 0
+149.171.160.183 - - [01/Jul/1995:00:24:37 -0400] "GET /shuttle/missions/sts-53/movies/sts-53-launch.mpg HTTP/1.0" 200 57344
+palona1.cns.hp.com - - [01/Jul/1995:00:24:39 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+alyssa.prodigy.com - - [01/Jul/1995:00:24:44 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:24:45 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+burger.letters.com - - [01/Jul/1995:00:24:45 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:24:45 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+burger.letters.com - - [01/Jul/1995:00:24:46 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:24:47 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+leo.nmc.edu - - [01/Jul/1995:00:24:48 -0400] "GET /shuttle/countdown/lps/fr.html HTTP/1.0" 200 1879
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:24:49 -0400] "GET /shuttle/missions/sts-67/mission-sts-67.html HTTP/1.0" 200 21408
+piweba3y.prodigy.com - - [01/Jul/1995:00:24:51 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:24:51 -0400] "GET /shuttle/missions/sts-67/sts-67-patch-small.gif HTTP/1.0" 200 17083
+alyssa.prodigy.com - - [01/Jul/1995:00:24:56 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+hoflink.com - - [01/Jul/1995:00:24:58 -0400] "GET /shuttle/missions/100th.html HTTP/1.0" 200 32303
+burger.letters.com - - [01/Jul/1995:00:24:59 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 70191
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:24:59 -0400] "GET /facilities/lc39a.html HTTP/1.0" 200 7008
+sartre.execpc.com - - [01/Jul/1995:00:25:00 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+boing.dgsys.com - - [01/Jul/1995:00:25:02 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+www-b3.proxy.aol.com - - [01/Jul/1995:00:25:02 -0400] "GET / HTTP/1.0" 304 0
+sartre.execpc.com - - [01/Jul/1995:00:25:02 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+indy2.indy.net - - [01/Jul/1995:00:25:02 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:25:03 -0400] "GET /images/lc39a-logo.gif HTTP/1.0" 200 13116
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:25:03 -0400] "GET /cgi-bin/imagemap/countdown?103,174 HTTP/1.0" 302 110
+boing.dgsys.com - - [01/Jul/1995:00:25:04 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:25:05 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+sartre.execpc.com - - [01/Jul/1995:00:25:06 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+indy2.indy.net - - [01/Jul/1995:00:25:08 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:25:08 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+149.171.160.183 - - [01/Jul/1995:00:25:08 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-tcdt-crew-walkout.mpg HTTP/1.0" 200 49152
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:25:10 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+ppp24.swcp.com - - [01/Jul/1995:00:25:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0423.gif HTTP/1.0" 200 64939
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:25:15 -0400] "GET /images/kscmap-tiny.gif HTTP/1.0" 200 2537
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:25:16 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:25:16 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:25:18 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+indy2.indy.net - - [01/Jul/1995:00:25:18 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+leo.nmc.edu - - [01/Jul/1995:00:25:19 -0400] "GET /shuttle/countdown/lps/omr/omr.html HTTP/1.0" 200 1341
+indy2.indy.net - - [01/Jul/1995:00:25:20 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+sartre.execpc.com - - [01/Jul/1995:00:25:20 -0400] "GET /shuttle/countdown/ HTTP/1.0" 304 0
+ix-phx5-17.ix.netcom.com - - [01/Jul/1995:00:25:21 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0747.txt HTTP/1.0" 200 1301
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:25:21 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:25:26 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+sartre.execpc.com - - [01/Jul/1995:00:25:27 -0400] "GET /cgi-bin/imagemap/countdown?100,108 HTTP/1.0" 302 111
+zoom112.telepath.com - - [01/Jul/1995:00:25:27 -0400] "GET /history/apollo-13/apollo-13.html HTTP/1.0" 404 -
+sartre.execpc.com - - [01/Jul/1995:00:25:27 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 304 0
+boing.dgsys.com - - [01/Jul/1995:00:25:29 -0400] "GET /shuttle/missions/sts-67/mission-sts-67.html HTTP/1.0" 200 21408
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:25:30 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0916.gif HTTP/1.0" 200 25814
+boing.dgsys.com - - [01/Jul/1995:00:25:31 -0400] "GET /shuttle/missions/sts-67/sts-67-patch-small.gif HTTP/1.0" 200 17083
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:25:33 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:25:34 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+hfx-p15.isisnet.com - - [01/Jul/1995:00:25:35 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:25:35 -0400] "GET /cgi-bin/imagemap/countdown?329,273 HTTP/1.0" 302 98
+indy2.indy.net - - [01/Jul/1995:00:25:35 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+149.171.160.182 - - [01/Jul/1995:00:25:35 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:25:36 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+149.171.160.182 - - [01/Jul/1995:00:25:37 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:25:38 -0400] "GET /history/apollo/apollo-1/apollo-1.html HTTP/1.0" 200 3842
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:25:38 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+indy2.indy.net - - [01/Jul/1995:00:25:39 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:25:39 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:25:39 -0400] "GET /history/apollo/apollo-1/apollo-1-patch-small.gif HTTP/1.0" 200 16979
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:25:42 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+hfx-p15.isisnet.com - - [01/Jul/1995:00:25:42 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+leo.nmc.edu - - [01/Jul/1995:00:25:43 -0400] "GET /shuttle/countdown/lps/fr.html HTTP/1.0" 200 1879
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:25:44 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+149.171.160.182 - - [01/Jul/1995:00:25:44 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:46 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:25:48 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:48 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+hfx-p15.isisnet.com - - [01/Jul/1995:00:25:49 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:50 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:51 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:51 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+n1031681.ksc.nasa.gov - - [01/Jul/1995:00:25:52 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+boing.dgsys.com - - [01/Jul/1995:00:25:52 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+teleman.pr.mcs.net - - [01/Jul/1995:00:25:53 -0400] "GET /history/apollo/apollo-13/sounds/a13_002.wav HTTP/1.0" 200 127045
+rlnport2.cin.dca.com - - [01/Jul/1995:00:25:54 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:25:57 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 72102
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:26:00 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0917.gif HTTP/1.0" 200 30995
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:03 -0400] "GET /shuttle/missions/sts-77/mission-sts-77.html HTTP/1.0" 200 3071
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:06 -0400] "GET /shuttle/missions/sts-77/sts-77-patch-small.gif HTTP/1.0" 200 4179
+boing.dgsys.com - - [01/Jul/1995:00:26:08 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+128.187.140.171 - - [01/Jul/1995:00:26:08 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+sartre.execpc.com - - [01/Jul/1995:00:26:08 -0400] "GET /shuttle/missions/sts-71/sts-71-press-kit.txt HTTP/1.0" 200 78588
+128.187.140.171 - - [01/Jul/1995:00:26:09 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+128.187.140.171 - - [01/Jul/1995:00:26:09 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+128.187.140.171 - - [01/Jul/1995:00:26:09 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+128.187.140.171 - - [01/Jul/1995:00:26:09 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+128.187.140.171 - - [01/Jul/1995:00:26:10 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+boing.dgsys.com - - [01/Jul/1995:00:26:10 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+128.187.140.171 - - [01/Jul/1995:00:26:11 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+remote1-p16.ume.maine.edu - - [01/Jul/1995:00:26:12 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+128.187.140.171 - - [01/Jul/1995:00:26:12 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+128.187.140.171 - - [01/Jul/1995:00:26:12 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:26:14 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+piweba3y.prodigy.com - - [01/Jul/1995:00:26:15 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0544.jpg HTTP/1.0" 200 70128
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:26:17 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:26:17 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+dnet018.sat.texas.net - - [01/Jul/1995:00:26:17 -0400] "GET /history/apollo/apollo-13/images/70HC323.GIF HTTP/1.0" 200 154223
+brandt.xensei.com - - [01/Jul/1995:00:26:18 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-mir-dock.mpg HTTP/1.0" 200 946425
+cruzio.com - - [01/Jul/1995:00:26:18 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+128.187.140.171 - - [01/Jul/1995:00:26:23 -0400] "GET /cgi-bin/imagemap/countdown?97,140 HTTP/1.0" 302 96
+cruzio.com - - [01/Jul/1995:00:26:23 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 47699
+www-b3.proxy.aol.com - - [01/Jul/1995:00:26:24 -0400] "GET /cgi-bin/imagemap/countdown?89,145 HTTP/1.0" 302 96
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:26:24 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+f180-049.net.wisc.edu - - [01/Jul/1995:00:26:25 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+cts25.cc.utah.edu - - [01/Jul/1995:00:26:25 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+dal31.pic.net - - [01/Jul/1995:00:26:25 -0400] "GET /shuttle/resources/orbiters/orbiters.html HTTP/1.0" 200 2178
+f180-049.net.wisc.edu - - [01/Jul/1995:00:26:26 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+hfx-p15.isisnet.com - - [01/Jul/1995:00:26:26 -0400] "GET /cgi-bin/imagemap/countdown?107,148 HTTP/1.0" 302 96
+dal31.pic.net - - [01/Jul/1995:00:26:28 -0400] "GET /images/landing-small.gif HTTP/1.0" 200 16966
+pm4_23.digital.net - - [01/Jul/1995:00:26:28 -0400] "GET /shuttle/technology/sts-newsref/stsref-toc.html HTTP/1.0" 200 81920
+dal31.pic.net - - [01/Jul/1995:00:26:28 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ppp1.niag.localnet.com - - [01/Jul/1995:00:26:28 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:31 -0400] "GET /shuttle/missions/sts-73/mission-sts-73.html HTTP/1.0" 200 4085
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:32 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:33 -0400] "GET /shuttle/missions/sts-73/sts-73-patch-small.gif HTTP/1.0" 200 4179
+remote9.compusmart.ab.ca - - [01/Jul/1995:00:26:35 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+fht.cts.com - - [01/Jul/1995:00:26:35 -0400] "GET /history/apollo/apollo.html HTTP/1.0" 200 3258
+fht.cts.com - - [01/Jul/1995:00:26:36 -0400] "GET /history/apollo/images/footprint-small.gif HTTP/1.0" 200 18149
+fht.cts.com - - [01/Jul/1995:00:26:37 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+fht.cts.com - - [01/Jul/1995:00:26:37 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+remote9.compusmart.ab.ca - - [01/Jul/1995:00:26:37 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:37 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:26:38 -0400] "GET /cgi-bin/imagemap/countdown?97,174 HTTP/1.0" 302 110
+ix-spr-ma1-01.ix.netcom.com - - [01/Jul/1995:00:26:39 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:26:40 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+n2o.phantom.com - - [01/Jul/1995:00:26:41 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+n2o.phantom.com - - [01/Jul/1995:00:26:44 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+n2o.phantom.com - - [01/Jul/1995:00:26:46 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:26:47 -0400] "GET /shuttle/resources/orbiters/atlantis.html HTTP/1.0" 200 7025
+f180-049.net.wisc.edu - - [01/Jul/1995:00:26:47 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 70165
+n2o.phantom.com - - [01/Jul/1995:00:26:48 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:49 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:50 -0400] "GET /shuttle/missions/sts-69/mission-sts-69.html HTTP/1.0" 200 4325
+annex12-36.dial.umd.edu - - [01/Jul/1995:00:26:50 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0893.gif HTTP/1.0" 200 49449
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:51 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:26:52 -0400] "GET /shuttle/missions/sts-69/sts-69-patch-small.gif HTTP/1.0" 200 8083
+remote9.compusmart.ab.ca - - [01/Jul/1995:00:26:52 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:26:52 -0400] "GET /shuttle/resources/orbiters/atlantis-logo.gif HTTP/1.0" 200 4179
+indy2.indy.net - - [01/Jul/1995:00:26:52 -0400] "GET /cgi-bin/imagemap/countdown?320,279 HTTP/1.0" 302 98
+cs2-07.all.ptd.net - - [01/Jul/1995:00:26:53 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+remote9.compusmart.ab.ca - - [01/Jul/1995:00:26:53 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+indy2.indy.net - - [01/Jul/1995:00:26:54 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+dal31.pic.net - - [01/Jul/1995:00:26:54 -0400] "GET /shuttle/resources/orbiters/enterprise.html HTTP/1.0" 200 9732
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:55 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+cs2-07.all.ptd.net - - [01/Jul/1995:00:26:56 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+remote1-p16.ume.maine.edu - - [01/Jul/1995:00:26:56 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:26:57 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+dal31.pic.net - - [01/Jul/1995:00:26:57 -0400] "GET /shuttle/resources/orbiters/enterprise-logo.gif HTTP/1.0" 200 25257
+cs2-07.all.ptd.net - - [01/Jul/1995:00:26:57 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+cs2-07.all.ptd.net - - [01/Jul/1995:00:26:57 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ix-wbg-va2-26.ix.netcom.com - - [01/Jul/1995:00:27:00 -0400] "GET /history/apollo/apollo-11/sounds/A01106AA.WAV HTTP/1.0" 200 96036
+dialup33.azstarnet.com - - [01/Jul/1995:00:27:02 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:27:02 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+dal31.pic.net - - [01/Jul/1995:00:27:04 -0400] "GET /images/landing-logo.gif HTTP/1.0" 200 2582
+fht.cts.com - - [01/Jul/1995:00:27:06 -0400] "GET /history/apollo/apollo-goals.txt HTTP/1.0" 200 712
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:27:07 -0400] "GET /shuttle/resources/orbiters/orbiters-logo.gif HTTP/1.0" 200 1932
+zoom112.telepath.com - - [01/Jul/1995:00:27:07 -0400] "GET /history/apollo/apollo-13/apollo-13.html HTTP/1.0" 200 18114
+zoom112.telepath.com - - [01/Jul/1995:00:27:09 -0400] "GET /history/apollo/apollo-13/apollo-13-patch-small.gif HTTP/1.0" 200 12859
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:27:09 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:27:11 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:27:13 -0400] "GET /history/apollo/apollo-7/apollo-7.html HTTP/1.0" 200 14136
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:27:14 -0400] "GET /shuttle/missions/sts-70/mission-sts-70.html HTTP/1.0" 200 13469
+yiura.dial-switch.ch - - [01/Jul/1995:00:27:14 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+remote1-p16.ume.maine.edu - - [01/Jul/1995:00:27:14 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:27:15 -0400] "GET /history/apollo/apollo-7/apollo-7-patch-small.gif HTTP/1.0" 200 13218
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:16 -0400] "GET / HTTP/1.0" 200 7074
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:27:16 -0400] "GET /shuttle/missions/sts-70/sts-70-patch-small.gif HTTP/1.0" 200 5026
+yiura.dial-switch.ch - - [01/Jul/1995:00:27:18 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+yiura.dial-switch.ch - - [01/Jul/1995:00:27:18 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+yiura.dial-switch.ch - - [01/Jul/1995:00:27:18 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+indy2.indy.net - - [01/Jul/1995:00:27:18 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 71743
+teleman.pr.mcs.net - - [01/Jul/1995:00:27:19 -0400] "GET /history/apollo/apollo-13/movies/ HTTP/1.0" 200 945
+dnet018.sat.texas.net - - [01/Jul/1995:00:27:22 -0400] "GET /history/apollo/apollo-13/images/index.gif HTTP/1.0" 200 99942
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:22 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+piweba3y.prodigy.com - - [01/Jul/1995:00:27:23 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+slip2-42.acs.ohio-state.edu - - [01/Jul/1995:00:27:24 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+teleman.pr.mcs.net - - [01/Jul/1995:00:27:24 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+dialup28.nmia.com - - [01/Jul/1995:00:27:24 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:27:25 -0400] "GET /history/apollo/apollo-1/apollo-1.html HTTP/1.0" 304 0
+zoom112.telepath.com - - [01/Jul/1995:00:27:26 -0400] "GET /history/apollo/images/footprint-logo.gif HTTP/1.0" 200 4209
+zoom112.telepath.com - - [01/Jul/1995:00:27:26 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+brandt.xensei.com - - [01/Jul/1995:00:27:27 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+dd11-006.compuserve.com - - [01/Jul/1995:00:27:27 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+dialup28.nmia.com - - [01/Jul/1995:00:27:28 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+dialup28.nmia.com - - [01/Jul/1995:00:27:29 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dialup28.nmia.com - - [01/Jul/1995:00:27:29 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:30 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+brandt.xensei.com - - [01/Jul/1995:00:27:31 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+www-b3.proxy.aol.com - - [01/Jul/1995:00:27:31 -0400] "GET /cgi-bin/imagemap/countdown?367,277 HTTP/1.0" 302 68
+202.33.84.230 - - [01/Jul/1995:00:27:31 -0400] "GET /history/mercury/mercury.html HTTP/1.0" 200 1871
+166.79.67.111 - - [01/Jul/1995:00:27:32 -0400] "GET /facilities/tour.html HTTP/1.0" 200 3723
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:27:32 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+fht.cts.com - - [01/Jul/1995:00:27:32 -0400] "GET /history/apollo/apollo-spacecraft.txt HTTP/1.0" 200 2261
+202.33.84.230 - - [01/Jul/1995:00:27:33 -0400] "GET /images/mercury-logo.gif HTTP/1.0" 200 6588
+202.33.84.230 - - [01/Jul/1995:00:27:33 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+202.33.84.230 - - [01/Jul/1995:00:27:33 -0400] "GET /history/apollo/images/apollo-logo.gif HTTP/1.0" 200 3047
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:27:34 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+f180-049.net.wisc.edu - - [01/Jul/1995:00:27:35 -0400] "GET /shuttle/countdown/video/livevideo.jpeg HTTP/1.0" 200 49425
+dd11-006.compuserve.com - - [01/Jul/1995:00:27:35 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:36 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+dialport-28.sh.lsumc.edu - - [01/Jul/1995:00:27:36 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+gclab040.ins.gu.edu.au - - [01/Jul/1995:00:27:37 -0400] "GET /htbin/cdt_clock.pl HTTP/1.0" 200 543
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:27:38 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+n2o.phantom.com - - [01/Jul/1995:00:27:39 -0400] "GET /cgi-bin/imagemap/countdown?107,175 HTTP/1.0" 302 110
+dialport-28.sh.lsumc.edu - - [01/Jul/1995:00:27:40 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+dialport-28.sh.lsumc.edu - - [01/Jul/1995:00:27:40 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+dialport-28.sh.lsumc.edu - - [01/Jul/1995:00:27:40 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+n2o.phantom.com - - [01/Jul/1995:00:27:40 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+dd11-006.compuserve.com - - [01/Jul/1995:00:27:43 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+dal08.onramp.net - - [01/Jul/1995:00:27:44 -0400] "GET /facts/facts.html HTTP/1.0" 200 4717
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:45 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+hoflink.com - - [01/Jul/1995:00:27:45 -0400] "GET /shuttle/missions/sts-5/mission-sts-5.html HTTP/1.0" 200 4995
+dialup33.azstarnet.com - - [01/Jul/1995:00:27:46 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0915.txt HTTP/1.0" 200 1389
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:27:46 -0400] "GET /history/apollo/as-203/as-203.html HTTP/1.0" 200 2424
+dal08.onramp.net - - [01/Jul/1995:00:27:46 -0400] "GET /images/faq.gif HTTP/1.0" 200 263
+202.33.84.230 - - [01/Jul/1995:00:27:46 -0400] "GET /history/mercury/mercury-goals.txt HTTP/1.0" 200 421
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:27:47 -0400] "GET / HTTP/1.0" 200 7074
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:27:48 -0400] "GET /history/apollo/as-203/as-203-patch-small.gif HTTP/1.0" 200 16187
+dal08.onramp.net - - [01/Jul/1995:00:27:48 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:27:49 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+128.187.140.171 - - [01/Jul/1995:00:27:49 -0400] "GET /cgi-bin/imagemap/countdown?375,269 HTTP/1.0" 302 68
+sartre.execpc.com - - [01/Jul/1995:00:27:50 -0400] "GET /shuttle/missions/sts-71/sts-71-day-01-highlights.html HTTP/1.0" 200 2722
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:27:52 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 200 5866
+yiura.dial-switch.ch - - [01/Jul/1995:00:27:53 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+palona1.cns.hp.com - - [01/Jul/1995:00:27:55 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+www-a1.proxy.aol.com - - [01/Jul/1995:00:27:55 -0400] "GET /facilities/opf.html HTTP/1.0" 200 2355
+brandt.xensei.com - - [01/Jul/1995:00:27:56 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 71743
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:27:56 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+halon.sybase.com - - [01/Jul/1995:00:27:57 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:27:57 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+piweba3y.prodigy.com - - [01/Jul/1995:00:27:58 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+teleman.pr.mcs.net - - [01/Jul/1995:00:27:58 -0400] "GET /history/apollo/apollo-13/movies/apo13damage.mpg HTTP/1.0" 200 65536
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:27:59 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+dal08.onramp.net - - [01/Jul/1995:00:27:59 -0400] "GET /facts/faq04.html HTTP/1.0" 200 27063
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:27:59 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+palona1.cns.hp.com - - [01/Jul/1995:00:28:00 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ix-atl12-02.ix.netcom.com - - [01/Jul/1995:00:28:00 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 200 669
+tip-mp6-ncs-16.stanford.edu - - [01/Jul/1995:00:28:00 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+www-a1.proxy.aol.com - - [01/Jul/1995:00:28:00 -0400] "GET /images/opf-logo.gif HTTP/1.0" 200 32511
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:28:02 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+halon.sybase.com - - [01/Jul/1995:00:28:02 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+halon.sybase.com - - [01/Jul/1995:00:28:02 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+remote1-p16.ume.maine.edu - - [01/Jul/1995:00:28:03 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+kuts5p06.cc.ukans.edu - - [01/Jul/1995:00:28:06 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+halon.sybase.com - - [01/Jul/1995:00:28:08 -0400] "GET /cgi-bin/imagemap/countdown?112,111 HTTP/1.0" 302 111
+sartre.execpc.com - - [01/Jul/1995:00:28:10 -0400] "GET /cgi-bin/imagemap/countdown?86,140 HTTP/1.0" 302 96
+halon.sybase.com - - [01/Jul/1995:00:28:10 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+kristina.az.com - - [01/Jul/1995:00:28:12 -0400] "GET /history/apollo/apollo-13/apollo-13-info.html HTTP/1.0" 200 1583
+halon.sybase.com - - [01/Jul/1995:00:28:14 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+dialup28.nmia.com - - [01/Jul/1995:00:28:15 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+dal08.onramp.net - - [01/Jul/1995:00:28:15 -0400] "GET /images/ksclogosmall.gif HTTP/1.0" 200 3635
+palona1.cns.hp.com - - [01/Jul/1995:00:28:16 -0400] "GET /cgi-bin/imagemap/countdown?321,279 HTTP/1.0" 302 98
+palona1.cns.hp.com - - [01/Jul/1995:00:28:17 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 200 4538
+eagle.co.la.ca.us - - [01/Jul/1995:00:28:20 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+palona1.cns.hp.com - - [01/Jul/1995:00:28:21 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 76706
+ix-den6-18.ix.netcom.com - - [01/Jul/1995:00:28:23 -0400] "GET /images/lc39a.gif HTTP/1.0" 200 257194
+hoflink.com - - [01/Jul/1995:00:28:24 -0400] "GET /shuttle/missions/sts-5/sts-5-info.html HTTP/1.0" 200 1405
+teleman.pr.mcs.net - - [01/Jul/1995:00:28:25 -0400] "GET /history/apollo/apollo-13/news/ HTTP/1.0" 200 377
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:28:26 -0400] "GET /history/apollo/apollo-7/apollo-7.html HTTP/1.0" 200 14136
+eagle.co.la.ca.us - - [01/Jul/1995:00:28:27 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+eagle.co.la.ca.us - - [01/Jul/1995:00:28:27 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ad08-027.compuserve.com - - [01/Jul/1995:00:28:27 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+svasu.extern.ucsd.edu - - [01/Jul/1995:00:28:28 -0400] "GET /history/apollo/apollo-7/apollo-7-patch-small.gif HTTP/1.0" 200 13218
+sartre.execpc.com - - [01/Jul/1995:00:28:30 -0400] "GET /cgi-bin/imagemap/countdown?98,169 HTTP/1.0" 302 110
+n2o.phantom.com - - [01/Jul/1995:00:28:31 -0400] "GET /cgi-bin/imagemap/countdown?101,147 HTTP/1.0" 302 96
+sartre.execpc.com - - [01/Jul/1995:00:28:31 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+149.171.160.182 - - [01/Jul/1995:00:28:33 -0400] "GET /shuttle/missions/sts-71/sts-71-day-04-highlights.html HTTP/1.0" 200 5544
+htlulx.htl-bw.ch - - [01/Jul/1995:00:28:34 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+gbol16.dct.com - - [01/Jul/1995:00:28:36 -0400] "GET /ksc.html HTTP/1.0" 200 7074
+eagle.co.la.ca.us - - [01/Jul/1995:00:28:37 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+gbol16.dct.com - - [01/Jul/1995:00:28:37 -0400] "GET /images/ksclogo-medium.gif HTTP/1.0" 304 0
+htlulx.htl-bw.ch - - [01/Jul/1995:00:28:38 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+gbol16.dct.com - - [01/Jul/1995:00:28:42 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+gbol16.dct.com - - [01/Jul/1995:00:28:42 -0400] "GET /images/MOSAIC-logosmall.gif HTTP/1.0" 200 363
+gbol16.dct.com - - [01/Jul/1995:00:28:42 -0400] "GET /images/WORLD-logosmall.gif HTTP/1.0" 304 0
+gbol16.dct.com - - [01/Jul/1995:00:28:42 -0400] "GET /images/USA-logosmall.gif HTTP/1.0" 200 234
+kristina.az.com - - [01/Jul/1995:00:28:42 -0400] "GET /history/apollo/apollo-13/images/ HTTP/1.0" 200 1851
+brandt.xensei.com - - [01/Jul/1995:00:28:43 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+teleman.pr.mcs.net - - [01/Jul/1995:00:28:43 -0400] "GET /history/apollo/apollo-13/images/ HTTP/1.0" 200 1851
+kristina.az.com - - [01/Jul/1995:00:28:44 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+kristina.az.com - - [01/Jul/1995:00:28:44 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+kristina.az.com - - [01/Jul/1995:00:28:44 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+whlane.cts.com - - [01/Jul/1995:00:28:44 -0400] "GET /cgi-bin/imagemap/countdown?99,215 HTTP/1.0" 302 95
+cs2-07.all.ptd.net - - [01/Jul/1995:00:28:44 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+dd11-006.compuserve.com - - [01/Jul/1995:00:28:44 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+htlulx.htl-bw.ch - - [01/Jul/1995:00:28:45 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+teleman.pr.mcs.net - - [01/Jul/1995:00:28:45 -0400] "GET /icons/image.xbm HTTP/1.0" 200 509
+whlane.cts.com - - [01/Jul/1995:00:28:45 -0400] "GET /shuttle/countdown/tour.html HTTP/1.0" 200 4347
+htlulx.htl-bw.ch - - [01/Jul/1995:00:28:46 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:28:46 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+whlane.cts.com - - [01/Jul/1995:00:28:47 -0400] "GET /images/KSC-94EC-412-small.gif HTTP/1.0" 200 20484
+dialup28.nmia.com - - [01/Jul/1995:00:28:50 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 76706
+halon.sybase.com - - [01/Jul/1995:00:28:50 -0400] "GET /cgi-bin/imagemap/countdown?102,179 HTTP/1.0" 302 110
+halon.sybase.com - - [01/Jul/1995:00:28:52 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+brandt.xensei.com - - [01/Jul/1995:00:28:54 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:28:54 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+dal08.onramp.net - - [01/Jul/1995:00:28:55 -0400] "GET /facts/faq07.html HTTP/1.0" 200 10877
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:28:55 -0400] "GET /shuttle/missions/sts-71/news HTTP/1.0" 302 -
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:28:56 -0400] "GET /shuttle/missions/sts-71/news/ HTTP/1.0" 200 3442
+spazan.cts.com - - [01/Jul/1995:00:28:57 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+news.ti.com - - [01/Jul/1995:00:28:57 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0911.gif HTTP/1.0" 200 31242
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:28:58 -0400] "GET /icons/blank.xbm HTTP/1.0" 200 509
+spazan.cts.com - - [01/Jul/1995:00:28:59 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:29:00 -0400] "GET /icons/menu.xbm HTTP/1.0" 200 527
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:29:00 -0400] "GET /icons/text.xbm HTTP/1.0" 200 527
+hoflink.com - - [01/Jul/1995:00:29:00 -0400] "GET /shuttle/missions/sts-5/movies/ HTTP/1.0" 200 375
+icagen.vnet.net - - [01/Jul/1995:00:29:00 -0400] "GET /software/winvn/winvn.html HTTP/1.0" 200 9867
+thimble-d229.sierra.net - - [01/Jul/1995:00:29:01 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+spazan.cts.com - - [01/Jul/1995:00:29:03 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+spazan.cts.com - - [01/Jul/1995:00:29:05 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+icagen.vnet.net - - [01/Jul/1995:00:29:06 -0400] "GET /software/winvn/winvn.gif HTTP/1.0" 200 25218
+icagen.vnet.net - - [01/Jul/1995:00:29:06 -0400] "GET /images/construct.gif HTTP/1.0" 200 1414
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:29:07 -0400] "GET /cgi-bin/imagemap/countdown?109,171 HTTP/1.0" 302 110
+eagle.co.la.ca.us - - [01/Jul/1995:00:29:07 -0400] "GET /cgi-bin/imagemap/countdown?103,110 HTTP/1.0" 302 111
+deimos.lpl.arizona.edu - - [01/Jul/1995:00:29:08 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+eagle.co.la.ca.us - - [01/Jul/1995:00:29:08 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:29:09 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:29:09 -0400] "GET /shuttle/missions/missions.html HTTP/1.0" 200 8677
+eagle.co.la.ca.us - - [01/Jul/1995:00:29:11 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+halon.sybase.com - - [01/Jul/1995:00:29:11 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+blv-pm2-ip16.halcyon.com - - [01/Jul/1995:00:29:11 -0400] "GET /shuttle/missions/sts-71/news/sts-71-mcc-06.txt HTTP/1.0" 200 2700
+sartre.execpc.com - - [01/Jul/1995:00:29:14 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+156.151.176.24 - - [01/Jul/1995:00:29:14 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+dd11-006.compuserve.com - - [01/Jul/1995:00:29:15 -0400] "GET /shuttle/missions/sts-71/news HTTP/1.0" 302 -
+ix-ftw-tx1-21.ix.netcom.com - - [01/Jul/1995:00:29:19 -0400] "GET /cgi-bin/imagemap/countdown?374,271 HTTP/1.0" 302 68
+brandt.xensei.com - - [01/Jul/1995:00:29:20 -0400] "GET /shuttle/countdown/video/livevideo.gif HTTP/1.0" 200 72666
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:29:20 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
+156.151.176.24 - - [01/Jul/1995:00:29:21 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+kristina.az.com - - [01/Jul/1995:00:29:22 -0400] "GET /history/apollo/apollo-13/images/70HC314.GIF HTTP/1.0" 200 101267
+eagle.co.la.ca.us - - [01/Jul/1995:00:29:24 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+icagen.vnet.net - - [01/Jul/1995:00:29:26 -0400] "GET /software/winvn/bluemarb.gif HTTP/1.0" 200 4441
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:29:26 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0443.txt HTTP/1.0" 200 948
+icagen.vnet.net - - [01/Jul/1995:00:29:28 -0400] "GET /software/winvn/wvsmall.gif HTTP/1.0" 200 13372
+ssandyg.scvnet.com - - [01/Jul/1995:00:29:30 -0400] "GET /shuttle/countdown HTTP/1.0" 302 -
+news.ti.com - - [01/Jul/1995:00:29:31 -0400] "GET /shuttle/missions/sts-71/movies/ HTTP/1.0" 200 3090
+ssandyg.scvnet.com - - [01/Jul/1995:00:29:32 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:29:33 -0400] "GET /shuttle/missions/sts-71/images/images.html HTTP/1.0" 200 7634
+spazan.cts.com - - [01/Jul/1995:00:29:34 -0400] "GET /shuttle/missions/sts-78/mission-sts-78.html HTTP/1.0" 200 4377
+hoflink.com - - [01/Jul/1995:00:29:34 -0400] "GET /shuttle/missions/sts-5/ HTTP/1.0" 200 1585
+149.171.160.182 - - [01/Jul/1995:00:29:35 -0400] "GET /htbin/wais.pl?IMAX HTTP/1.0" 200 6923
+ssandyg.scvnet.com - - [01/Jul/1995:00:29:35 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ssandyg.scvnet.com - - [01/Jul/1995:00:29:35 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ssandyg.scvnet.com - - [01/Jul/1995:00:29:35 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+news.ti.com - - [01/Jul/1995:00:29:36 -0400] "GET /icons/movie.xbm HTTP/1.0" 200 530
+spazan.cts.com - - [01/Jul/1995:00:29:37 -0400] "GET /shuttle/missions/sts-78/sts-78-patch-small.gif HTTP/1.0" 200 4179
+halon.sybase.com - - [01/Jul/1995:00:29:39 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.gif HTTP/1.0" 200 31631
+thimble-d229.sierra.net - - [01/Jul/1995:00:29:40 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+spazan.cts.com - - [01/Jul/1995:00:29:41 -0400] "GET /images/launch-logo.gif HTTP/1.0" 200 1713
+sagami2.isc.meiji.ac.jp - - [01/Jul/1995:00:29:43 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+cs2-07.all.ptd.net - - [01/Jul/1995:00:29:48 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+mcmed4.med.nyu.edu - - [01/Jul/1995:00:29:49 -0400] "GET /shuttle/missions/sts-71/movies/crew-arrival-t38.mpg HTTP/1.0" 200 305722
+www-a1.proxy.aol.com - - [01/Jul/1995:00:29:49 -0400] "GET /facilities/lcc.html HTTP/1.0" 200 2489
+teleman.pr.mcs.net - - [01/Jul/1995:00:29:50 -0400] "GET /history/apollo/apollo-13/images/index.gif HTTP/1.0" 200 99942
+149.171.160.183 - - [01/Jul/1995:00:29:53 -0400] "GET /shuttle/missions/sts-71/movies/sts-71-launch-2.mpg HTTP/1.0" 200 262144
+www-a1.proxy.aol.com - - [01/Jul/1995:00:29:55 -0400] "GET /images/lcc-small2.gif HTTP/1.0" 200 58026
+ix-sd9-18.ix.netcom.com - - [01/Jul/1995:00:29:56 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0544.txt HTTP/1.0" 200 979
+icagen.vnet.net - - [01/Jul/1995:00:29:56 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+166.79.67.111 - - [01/Jul/1995:00:29:58 -0400] "GET /facilities/tour.html HTTP/1.0" 200 3723
+ip16-085.phx.primenet.com - - [01/Jul/1995:00:30:00 -0400] "GET /shuttle/countdown/countdown.html HTTP/1.0" 200 3985
+brandt.xensei.com - - [01/Jul/1995:00:30:00 -0400] "GET /shuttle/countdown/liftoff.html HTTP/1.0" 304 0
+ix-orl2-16.ix.netcom.com - - [01/Jul/1995:00:30:01 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.jpg HTTP/1.0" 200 46888
+ppp31.cowan.edu.au - - [01/Jul/1995:00:30:03 -0400] "GET /shuttle/countdown/ HTTP/1.0" 200 3985
+brandt.xensei.com - - [01/Jul/1995:00:30:05 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 304 0
+ssandyg.scvnet.com - - [01/Jul/1995:00:30:05 -0400] "GET /shuttle/missions/sts-71/movies/movies.html HTTP/1.0" 200 3092
+ppp31.cowan.edu.au - - [01/Jul/1995:00:30:06 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ip16-085.phx.primenet.com - - [01/Jul/1995:00:30:06 -0400] "GET /shuttle/countdown/count.gif HTTP/1.0" 200 40310
+ssandyg.scvnet.com - - [01/Jul/1995:00:30:07 -0400] "GET /shuttle/missions/sts-71/sts-71-patch-small.gif HTTP/1.0" 200 12054
+ip16-085.phx.primenet.com - - [01/Jul/1995:00:30:07 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786
+ppp31.cowan.edu.au - - [01/Jul/1995:00:30:09 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+ip16-085.phx.primenet.com - - [01/Jul/1995:00:30:10 -0400] "GET /images/KSC-logosmall.gif HTTP/1.0" 200 1204
+131.128.2.155 - - [01/Jul/1995:00:30:10 -0400] "GET /shuttle/missions/sts-71/mission-sts-71.html HTTP/1.0" 200 12040
+halon.sybase.com - - [01/Jul/1995:00:30:10 -0400] "GET /shuttle/missions/sts-71/images/KSC-95EC-0918.txt HTTP/1.0" 200 1391
+ppp31.cowan.edu.au - - [01/Jul/1995:00:30:14 -0400] "GET /images/NASA-logosmall.gif HTTP/1.0" 200 786

From pypy.commits at gmail.com  Mon Jun 20 07:04:48 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:48 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add perlin-noise config
Message-ID: <5767cdd0.8159c20a.cf0c1.ffff96de@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r359:9dd673d818fb
Date: 2016-05-28 09:29 +0200
http://bitbucket.org/pypy/benchmarks/changeset/9dd673d818fb/

Log:	add perlin-noise config

diff --git a/multithread/config-all-short.json b/multithread/config-all-short.json
--- a/multithread/config-all-short.json
+++ b/multithread/config-all-short.json
@@ -55,6 +55,12 @@
             "file": "raytrace/raytrace.py",
             "PYTHONPATH": "..",
             "args": ["128", "512"]
+        },
+
+        "perlin": {
+            "file": "perlin_noise/perlin_noise.py",
+            "PYTHONPATH": "..",
+            "args": ["4"]
         }
 
     }
diff --git a/multithread/config-perlin-noise.json b/multithread/config-perlin-noise.json
new file mode 100644
--- /dev/null
+++ b/multithread/config-perlin-noise.json
@@ -0,0 +1,19 @@
+{
+    "defaults": {
+        "file": null,
+        "threads": [1, 2, 4, 8],
+        "vmstarts": 3,
+        "warmiters": 5,
+        "PYTHONPATH": ".",
+        "args": [],
+        "cwd": "."
+    },
+
+    "benchs": {
+        "perlin": {
+            "file": "perlin_noise/perlin_noise.py",
+            "PYTHONPATH": "..",
+            "args": ["5"]
+        }
+    }
+}
diff --git a/multithread/perlin_noise/perlin_noise.py b/multithread/perlin_noise/perlin_noise.py
--- a/multithread/perlin_noise/perlin_noise.py
+++ b/multithread/perlin_noise/perlin_noise.py
@@ -3,7 +3,7 @@
 
 import sys
 import time, random
-from common.abstract_threading import (AtomicFuture,
+from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
     hint_commit_soon, print_abort_info)
 
@@ -67,16 +67,15 @@
 
 def work(n, x):
     res = []
-    hint_commit_soon()
-    with atomic:
-        for y in xrange(n):
-            for z in xrange(n):
-                res.append(perlin_noise(x/3., y/3., z/3.))
-    hint_commit_soon()
+    #hint_commit_soon()
+    for y in xrange(n):
+        for z in xrange(n):
+            res.append(perlin_noise(x/3., y/3., z/3.))
+    #hint_commit_soon()
     return res
 
 
-def run(threads=2, n=50):
+def run(threads=2, n=5):
     threads = int(threads)
     n = int(n)
 
@@ -91,5 +90,32 @@
     set_thread_pool(None)
 
 
+def main(argv):
+    # warmiters threads args...
+    warmiters = int(argv[0])
+    threads = int(argv[1])
+    n = int(argv[2])
+
+    print "params (iters, threads, n):", warmiters, threads, n
+
+    print "do warmup:"
+    for i in range(2):
+        t = time.time()
+        run(threads, n)
+        print "iter", i, "time:", time.time() - t
+
+    print "turn off jitting"
+    import gc
+    turn_jitting_off()
+    print "do", warmiters, "real iters:"
+    times = []
+    for i in range(warmiters):
+        gc.collect()
+        t = time.time()
+        run(threads, n)
+        times.append(time.time() - t)
+    print "warmiters:", times
+
 if __name__ == "__main__":
-    run()
+    import sys
+    main(sys.argv[1:])

From pypy.commits at gmail.com  Mon Jun 20 07:04:50 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:50 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: fix and start of
 get_results script
Message-ID: <5767cdd2.85c11c0a.26298.3d0e@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r360:c066151a4c13
Date: 2016-05-28 10:07 +0200
http://bitbucket.org/pypy/benchmarks/changeset/c066151a4c13/

Log:	fix and start of get_results script

diff --git a/multithread/get_results.py b/multithread/get_results.py
new file mode 100755
--- /dev/null
+++ b/multithread/get_results.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+import os
+import sys
+import json
+
+
+
+def main(argv):
+    results_file = argv[0]
+    assert os.path.exists(results_file)
+
+    with open(results_file, 'r') as f:
+        results = json.loads(f.read())
+
+    print "RUNS:", len(results)
+    for run_key, run in results.iteritems():
+        res_over_run = {}
+
+        print "###", "RUN", run_key, "###"
+        print "python:", run['python']
+        print "python-version:", run['python-version'].strip()
+        print "hg-id:", run['hg-id'].strip()
+        run_results = run['results']
+        print "RESULTS:", len(run_results)
+
+        print "BENCHMARKS:", run_results.keys()
+        for bench_key, bench_res in run_results.items():
+            print "BENCHMARK:", bench_key
+            if 'fail_reason' in bench_res:
+                print "FAILED:", bench_res
+            else:
+                timings = bench_res['timings']
+                failures = bench_res['failures']
+                print "timings:", len(timings), "failures:", len(failures)
+                res_over_run.setdefault(bench_key, []).extend(timings)
+                if failures:
+                    print "############# THERE ARE FAILURES! #############"
+                    #print "fail reasons:", failures
+                    #import pdb;pdb.set_trace()
+        print ""
+        print "RESULTS OF RUN:"
+        for bench_key, timings in res_over_run.items():
+            print "BENCH", bench_key
+            # group timings by thread (multiple vmstarts)
+            threads = results[run_key]['config']['defaults']['threads']
+            grouped = [[] for _ in range(max(threads))]
+            for t in timings:
+                grouped[t['threads'] - 1].extend(t['warmiters'])
+            for ts in threads:
+                print "TS:", ts, "times:", grouped[ts - 1]
+        print ""
+        print ""
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/multithread/perlin_noise/perlin_noise.py b/multithread/perlin_noise/perlin_noise.py
--- a/multithread/perlin_noise/perlin_noise.py
+++ b/multithread/perlin_noise/perlin_noise.py
@@ -5,7 +5,7 @@
 import time, random
 from common.abstract_threading import (
     atomic, Future, set_thread_pool, ThreadPool,
-    hint_commit_soon, print_abort_info)
+    hint_commit_soon, print_abort_info, turn_jitting_off)
 
 import itertools
 from collections import deque

From pypy.commits at gmail.com  Mon Jun 20 07:04:52 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:52 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: fix and adapt
 parameters for a specific machine
Message-ID: <5767cdd4.88c11c0a.7d2dc.fffffb1b@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r361:a6c5a631b030
Date: 2016-05-28 10:19 +0200
http://bitbucket.org/pypy/benchmarks/changeset/a6c5a631b030/

Log:	fix and adapt parameters for a specific machine

diff --git a/multithread/config-all-short.json b/multithread/config-all-short.json
--- a/multithread/config-all-short.json
+++ b/multithread/config-all-short.json
@@ -13,7 +13,7 @@
         "nqueens": {
             "file": "nqueens/nqueens.py",
             "PYTHONPATH": "..",
-            "args": ["10"]
+            "args": ["9"]
         },
 
         "parsible-bench": {
@@ -36,7 +36,7 @@
         "mandelbrot": {
             "file": "mandelbrot/mandelbrot.py",
             "PYTHONPATH": "..",
-            "args": ["64", "128", "512"]
+            "args": ["64", "384", "512"]
         },
 
         "btree": {
diff --git a/multithread/nqueens/nqueens.py b/multithread/nqueens/nqueens.py
--- a/multithread/nqueens/nqueens.py
+++ b/multithread/nqueens/nqueens.py
@@ -40,7 +40,7 @@
     solutions = []
     fs = []
     cols = range(n)
-    for perms in chunks(permutations(cols), 100000):
+    for perms in chunks(permutations(cols), 10000):
         fs.append(Future(check_solutions, n, cols, perms))
     print "Futures:", len(fs)
     for f in fs:
diff --git a/multithread/skiplist/skiplist.py b/multithread/skiplist/skiplist.py
--- a/multithread/skiplist/skiplist.py
+++ b/multithread/skiplist/skiplist.py
@@ -3,7 +3,7 @@
 
 from common.abstract_threading import (atomic, Future,
                                        set_thread_pool, ThreadPool,
-                                       hint_commit_soon)
+                                       hint_commit_soon, turn_jitting_off)
 import time, threading
 
 import random

From pypy.commits at gmail.com  Mon Jun 20 07:04:53 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:53 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: make get_results.py
 output a latex table for the paper
Message-ID: <5767cdd5.49c51c0a.69fe2.ffffe650@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r362:b4ffba47bd6a
Date: 2016-05-28 13:31 +0200
http://bitbucket.org/pypy/benchmarks/changeset/b4ffba47bd6a/

Log:	make get_results.py output a latex table for the paper

diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -3,17 +3,22 @@
 import os
 import sys
 import json
+import numpy as np
 
+def collect_warmiters(all_res, run_key):
+    # group timings by thread (multiple vmstarts)
+    res_over_run = all_res[run_key]
+    grouped = {}
+    for bench_key, timings in res_over_run.items():
+        for t in timings:
+            per_bench = grouped.setdefault(bench_key, {})
+            per_bench.setdefault(t['threads'], []).extend(t['warmiters'])
 
+    return grouped
 
-def main(argv):
-    results_file = argv[0]
-    assert os.path.exists(results_file)
-
-    with open(results_file, 'r') as f:
-        results = json.loads(f.read())
-
+def retrieve_data(results):
     print "RUNS:", len(results)
+    all_res = {}
     for run_key, run in results.iteritems():
         res_over_run = {}
 
@@ -38,19 +43,117 @@
                     print "############# THERE ARE FAILURES! #############"
                     #print "fail reasons:", failures
                     #import pdb;pdb.set_trace()
+        # print ""
+        # print "RESULTS OF RUN:"
+        all_res[run_key] = res_over_run
+        # grouped = collect_warmiters(all_res, run_key)
+        # for b, per_bench in grouped.items():
+        #     print "BENCH", b
+        #     for ts in sorted(per_bench.keys()):
+        #         if per_bench[ts]:
+        #             print "TS:", ts, "times:", per_bench[ts]
         print ""
-        print "RESULTS OF RUN:"
-        for bench_key, timings in res_over_run.items():
-            print "BENCH", bench_key
-            # group timings by thread (multiple vmstarts)
-            threads = results[run_key]['config']['defaults']['threads']
-            grouped = [[] for _ in range(max(threads))]
-            for t in timings:
-                grouped[t['threads'] - 1].extend(t['warmiters'])
+        print ""
+    return all_res
+
+
+def print_csv(all_res, run_key):
+    import numpy as np
+
+    grouped = collect_warmiters(all_res, run_key)
+    cols = len(grouped) * 2 + 1 # "threads" + avg, stddev for each benchmark
+    rows = len(grouped.values()[0]) + 1 # name + threads
+    table = [["" for _ in range(cols)] for _ in range(rows)] # t[row][col]
+
+    table[0][0] = "Threads"
+    for bench_num, (b, per_bench) in enumerate(grouped.items()):
+        print "BENCH", b
+        table[0][1 + bench_num * 2] = b
+        ts_index = 0
+        for ts in sorted(per_bench.keys()):
+            if per_bench[ts]:
+                row = 1 + ts_index
+                if table[row][0] == "":
+                    table[row][0] = ts
+                else:
+                    assert table[row][0] == ts
+
+                print "TS:", ts, "times:", per_bench[ts]
+                col = 1 + bench_num * 2
+                table[row][col] = np.mean(per_bench[ts])
+                table[row][col+1] = np.std(per_bench[ts])
+                ts_index += 1
+
+    # print table:
+    for r in range(rows):
+        line = ",\t".join(map(str, table[r]))
+        print line
+
+def print_latex_table(all_res, gil_run_key, stm_run_key):
+    print ""
+    print r"\footnotesize"
+    print r"\begin{tabularx}{\textwidth}{l|rrrr|rrrr|r}"
+    #print r"\hline"
+    print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} & \multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{1cm}}{\textbf{Max. speedup}} \\ \hline"
+    print r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline"
+
+    gil_grouped = collect_warmiters(all_res, gil_run_key)
+    stm_grouped = collect_warmiters(all_res, stm_run_key)
+
+    assert stm_grouped.keys() == gil_grouped.keys()
+    for bench_key in stm_grouped.keys():
+        elems = []
+        gil_bench = gil_grouped[bench_key]
+        stm_bench = stm_grouped[bench_key]
+        threads = [1, 2, 4, 8]
+        min_gil, min_stm = 9999999., 9999999.
+        for vm_bench in [gil_bench, stm_bench]:
             for ts in threads:
-                print "TS:", ts, "times:", grouped[ts - 1]
-        print ""
-        print ""
+                mean, std = np.mean(vm_bench[ts]), np.std(vm_bench[ts])
+                if vm_bench is gil_bench:
+                    if mean < min_gil:
+                        min_gil = mean
+                else:
+                    if mean < min_stm:
+                        min_stm = mean
+                elems.append((mean, std))
+
+        cells = []
+        min_mean = min(min_gil, min_stm)
+        for e in elems:
+            if e[0] == min_mean:
+                s = r"$\mathbf{%.2f}$ \scriptsize $\pm %.1f$ \footnotesize" % e
+            else:
+                s = r"$%.2f$ \scriptsize $\pm %.1f$ \footnotesize" % e
+            cells.append(s)
+        #
+        speedup = min_stm / min_gil
+        cells.append(r"$%.1f\times$" % speedup)
+        print r"%s & " % bench_key + " & ".join(cells) + r" \\"
+    #print r"\hline"
+    print r"\end{tabularx}"
+    print r"\normalsize"
+    print ""
+
+def main(argv):
+    results_file = argv[0]
+    assert os.path.exists(results_file)
+
+    with open(results_file, 'r') as f:
+        results = json.loads(f.read())
+
+    all_res = retrieve_data(results)
+    while True:
+        print "select gil and stm run (e.g., 0,1):"
+        runs = all_res.keys()
+        choices = ["%s: %s (%s)" % (i, r, results[r]['python'])
+                   for i, r in enumerate(runs)]
+        print "\n".join(choices)
+        choice = raw_input()
+
+        gil_run_key, stm_run_key = [runs[int(c)] for c in choice.split(',')]
+        #print_csv(all_res, stm_run_key)
+        print_latex_table(all_res, gil_run_key, stm_run_key)
 
 
 if __name__ == '__main__':

From pypy.commits at gmail.com  Mon Jun 20 07:04:55 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:55 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: fix speedup...
Message-ID: <5767cdd7.4a9bc20a.b68d3.43e6@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r363:bf55a0a552d0
Date: 2016-05-28 15:31 +0200
http://bitbucket.org/pypy/benchmarks/changeset/bf55a0a552d0/

Log:	fix speedup...

diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -92,7 +92,7 @@
 def print_latex_table(all_res, gil_run_key, stm_run_key):
     print ""
     print r"\footnotesize"
-    print r"\begin{tabularx}{\textwidth}{l|rrrr|rrrr|r}"
+    print r"\begin{tabularx}{\textwidth}{l|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r}"
     #print r"\hline"
     print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} & \multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{1cm}}{\textbf{Max. speedup}} \\ \hline"
     print r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline"
@@ -127,10 +127,10 @@
                 s = r"$%.2f$ \scriptsize $\pm %.1f$ \footnotesize" % e
             cells.append(s)
         #
-        speedup = min_stm / min_gil
+        speedup = min_gil / min_stm
         cells.append(r"$%.1f\times$" % speedup)
         print r"%s & " % bench_key + " & ".join(cells) + r" \\"
-    #print r"\hline"
+    print r"\hline"
     print r"\end{tabularx}"
     print r"\normalsize"
     print ""

From pypy.commits at gmail.com  Mon Jun 20 07:04:57 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:57 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add numactl and
 improvements to get_results
Message-ID: <5767cdd9.50991c0a.9b0b0.ffff9055@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r364:919a550f3977
Date: 2016-06-02 14:23 +0200
http://bitbucket.org/pypy/benchmarks/changeset/919a550f3977/

Log:	add numactl and improvements to get_results

diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -101,6 +101,7 @@
     stm_grouped = collect_warmiters(all_res, stm_run_key)
 
     assert stm_grouped.keys() == gil_grouped.keys()
+    warnings = ""
     for bench_key in stm_grouped.keys():
         elems = []
         gil_bench = gil_grouped[bench_key]
@@ -109,7 +110,10 @@
         min_gil, min_stm = 9999999., 9999999.
         for vm_bench in [gil_bench, stm_bench]:
             for ts in threads:
-                mean, std = np.mean(vm_bench[ts]), np.std(vm_bench[ts])
+                samples = vm_bench[ts]
+                if len(samples) < 30:
+                    warnings += "WARNING, %s had only %s samples\n" % (bench_key, len(samples))
+                mean, std = np.mean(samples), np.std(samples)
                 if vm_bench is gil_bench:
                     if mean < min_gil:
                         min_gil = mean
@@ -121,19 +125,20 @@
         cells = []
         min_mean = min(min_gil, min_stm)
         for e in elems:
-            if e[0] == min_mean:
+            if e[0] == min_gil or e[0] == min_stm:
                 s = r"$\mathbf{%.2f}$ \scriptsize $\pm %.1f$ \footnotesize" % e
             else:
                 s = r"$%.2f$ \scriptsize $\pm %.1f$ \footnotesize" % e
             cells.append(s)
         #
         speedup = min_gil / min_stm
-        cells.append(r"$%.1f\times$" % speedup)
+        cells.append(r"$%.2f\times$" % speedup)
         print r"%s & " % bench_key + " & ".join(cells) + r" \\"
     print r"\hline"
     print r"\end{tabularx}"
     print r"\normalsize"
     print ""
+    print warnings
 
 def main(argv):
     results_file = argv[0]
@@ -152,7 +157,7 @@
         choice = raw_input()
 
         gil_run_key, stm_run_key = [runs[int(c)] for c in choice.split(',')]
-        #print_csv(all_res, stm_run_key)
+        print_csv(all_res, gil_run_key)
         print_latex_table(all_res, gil_run_key, stm_run_key)
 
 
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -9,6 +9,7 @@
 import pprint
 from subprocess import Popen, PIPE
 
+WITH_NUMACTL = True
 
 def extract_iter_times(stdout):
     for line in stdout.split('\n'):
@@ -35,6 +36,9 @@
                     str(bench_config['warmiters']),
                     str(ts)]
                    + bench_config['args'])
+            if WITH_NUMACTL:
+                # run on node 2, allocate preferrably on node 2
+                cmd = ["numactl", "-N2", "--preferred=2"] + cmd
             cmd_str = " ".join(cmd)
 
             cwd, _ = os.path.split(bench_file)

From pypy.commits at gmail.com  Mon Jun 20 07:04:59 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:04:59 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: fix a few benchmarks
 and better configuration
Message-ID: <5767cddb.c8981c0a.81009.2de5@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r365:a417ae48b64d
Date: 2016-06-02 15:46 +0200
http://bitbucket.org/pypy/benchmarks/changeset/a417ae48b64d/

Log:	fix a few benchmarks and better configuration

diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -128,6 +128,9 @@
         for w in self.workers:
             w.join()
 
+    def __str__(self):
+        return "TP(%s)" % len(self.workers)
+
 
 _thread_pool = ThreadPool()
 atexit.register(_thread_pool.shutdown)
@@ -136,6 +139,7 @@
     global _thread_pool
     if _thread_pool:
         _thread_pool.shutdown()
+    print "set_thread_pool", th
     _thread_pool = th
 
 
diff --git a/multithread/config-all-short.json b/multithread/config-all-short.json
--- a/multithread/config-all-short.json
+++ b/multithread/config-all-short.json
@@ -24,7 +24,7 @@
         "mersenne": {
             "file": "mersenne/mersenne.py",
             "PYTHONPATH": "..",
-            "args": ["1500"]
+            "args": ["2000"]
         },
 
         "richards": {
@@ -48,7 +48,7 @@
         "skiplist": {
             "file": "skiplist/skiplist.py",
             "PYTHONPATH": "..",
-            "args": ["16", "100000"]
+            "args": ["16", "200000"]
         },
 
         "raytrace": {
diff --git a/multithread/mersenne/mersenne.py b/multithread/mersenne/mersenne.py
--- a/multithread/mersenne/mersenne.py
+++ b/multithread/mersenne/mersenne.py
@@ -73,7 +73,9 @@
 
     counter = [0]
     fs = []
-    for ps in chunks(xrange(2, upb_prime+1), 500):
+    cs = list(chunks(xrange(2, upb_prime+1), 100))
+    print len(cs), "futures"
+    for ps in cs:
         fs.append(Future(work, ps, counter, upb_count))
 
     [f() for f in fs]
diff --git a/multithread/nqueens/nqueens.py b/multithread/nqueens/nqueens.py
--- a/multithread/nqueens/nqueens.py
+++ b/multithread/nqueens/nqueens.py
@@ -6,7 +6,7 @@
 # of the execution time constructing the permutations. Thus
 # only half the execution is parallelised.
 
-
+import gc
 import sys
 import time
 from common.abstract_threading import (
@@ -40,13 +40,17 @@
     solutions = []
     fs = []
     cols = range(n)
-    for perms in chunks(permutations(cols), 10000):
+    cs = list(chunks(permutations(cols), 10000))
+    gc.collect()
+    t = time.time()
+    for perms in cs:
         fs.append(Future(check_solutions, n, cols, perms))
     print "Futures:", len(fs)
     for f in fs:
         solutions.extend(f())
-
+    t = time.time() - t
     print "found:", len(solutions)
+    return t
 
 
 def run(threads=2, n=10):
@@ -55,11 +59,11 @@
 
     set_thread_pool(ThreadPool(threads))
 
-    find_solutions(n)
+    t = find_solutions(n)
 
     # shutdown current pool
     set_thread_pool(None)
-
+    return t
 
 def main(argv):
     # warmiters threads args...
@@ -71,20 +75,18 @@
 
     print "do warmup:"
     for i in range(3):
-        t = time.time()
-        run(threads, n)
-        print "iter", i, "time:", time.time() - t
+        t = run(threads, n)
+        print "iter", i, "time:", t
 
     print "turn off jitting"
-    import gc
     turn_jitting_off()
     print "do", warmiters, "real iters:"
     times = []
     for i in range(warmiters):
         gc.collect()
-        t = time.time()
-        run(threads, n)
-        times.append(time.time() - t)
+
+        t = run(threads, n)
+        times.append(t)
     print "warmiters:", times
 
 if __name__ == "__main__":

From pypy.commits at gmail.com  Mon Jun 20 07:05:02 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:05:02 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: add retrying to
 runner.py
Message-ID: <5767cdde.e976c20a.17417.ffffb207@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r367:8b3d8eca2c1b
Date: 2016-06-02 15:56 +0200
http://bitbucket.org/pypy/benchmarks/changeset/8b3d8eca2c1b/

Log:	add retrying to runner.py

diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -10,6 +10,7 @@
 from subprocess import Popen, PIPE
 
 WITH_NUMACTL = True
+MAX_RETRY = 100  # per bench
 
 def extract_iter_times(stdout):
     for line in stdout.split('\n'):
@@ -26,8 +27,10 @@
 
     failures = []
     timings = []
+    retries = 0
     for ts in threads:
-        for vm in range(vmstarts):
+        vm = 0
+        while vm < vmstarts:
             print "threads: %s, vm: %s" % (ts, vm)
 
             bench_file = os.path.abspath(bench_config['file'])
@@ -68,6 +71,10 @@
                     }
                     failures.append(failure)
                     print "failure:", failure
+                    if retries < MAX_RETRY:
+                        print "RETRY"
+                        retries += 1
+                        continue  # w/o incrementing 'vm'
                 else:
                     stdout, stderr = p.stdout.read(), p.stderr.read()
                     print stdout
@@ -89,6 +96,7 @@
                 failures.append({
                     'cmd': cmd_str, 'exception': 'KeyboardInterrupt'})
                 return failures, timings
+            vm += 1
     return failures, timings
 
 

From pypy.commits at gmail.com  Mon Jun 20 07:05:04 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:05:04 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: cosmetics
Message-ID: <5767cde0.e7c9c20a.9dfba.ffffa18f@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r368:7ab40ed88522
Date: 2016-06-20 13:01 +0200
http://bitbucket.org/pypy/benchmarks/changeset/7ab40ed88522/

Log:	cosmetics

diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -94,7 +94,7 @@
     print r"\footnotesize"
     print r"\begin{tabularx}{\textwidth}{l|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r}"
     #print r"\hline"
-    print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} & \multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{1cm}}{\textbf{Max. speedup}} \\ \hline"
+    print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} & \multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{2cm}}{\textbf{Max. speedup}} \\ \hline"
     print r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline"
 
     gil_grouped = collect_warmiters(all_res, gil_run_key)
@@ -102,6 +102,7 @@
 
     assert stm_grouped.keys() == gil_grouped.keys()
     warnings = ""
+    lines = 1
     for bench_key in stm_grouped.keys():
         elems = []
         gil_bench = gil_grouped[bench_key]
@@ -126,14 +127,16 @@
         min_mean = min(min_gil, min_stm)
         for e in elems:
             if e[0] == min_gil or e[0] == min_stm:
-                s = r"$\mathbf{%.2f}$ \scriptsize $\pm %.1f$ \footnotesize" % e
+                s = r"$\mathbf{%.2f}$ \tiny $\pm %.1f$ \footnotesize" % e
             else:
-                s = r"$%.2f$ \scriptsize $\pm %.1f$ \footnotesize" % e
+                s = r"$%.2f$ \tiny $\pm %.1f$ \footnotesize" % e
             cells.append(s)
         #
         speedup = min_gil / min_stm
-        cells.append(r"$%.2f\times$" % speedup)
-        print r"%s & " % bench_key + " & ".join(cells) + r" \\"
+        cells.append(r"\multicolumn{1}{c}{$%.2f\times$}" % speedup)
+        print r"%s & " % bench_key + " & ".join(cells) + r" \\" + (
+            r" \hdashline[0.5pt/5pt]{}" if lines % 3 == 0 else "")
+        lines += 1
     print r"\hline"
     print r"\end{tabularx}"
     print r"\normalsize"
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -72,7 +72,8 @@
                     failures.append(failure)
                     print "failure:", failure
                     if retries < MAX_RETRY:
-                        print "RETRY"
+                        print "####### FAILURE! RETRYING #######"
+                        time.sleep(5)
                         retries += 1
                         continue  # w/o incrementing 'vm'
                 else:

From pypy.commits at gmail.com  Mon Jun 20 07:05:00 2016
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Jun 2016 04:05:00 -0700 (PDT)
Subject: [pypy-commit] benchmarks multithread-runner: merge
Message-ID: <5767cddc.cf2d1c0a.1c4a7.25d0@mx.google.com>

Author: Remi Meier 
Branch: multithread-runner
Changeset: r366:b4b77bd8fa7a
Date: 2016-06-02 15:47 +0200
http://bitbucket.org/pypy/benchmarks/changeset/b4b77bd8fa7a/

Log:	merge

diff --git a/multithread/get_results.py b/multithread/get_results.py
--- a/multithread/get_results.py
+++ b/multithread/get_results.py
@@ -3,17 +3,22 @@
 import os
 import sys
 import json
+import numpy as np
 
+def collect_warmiters(all_res, run_key):
+    # group timings by thread (multiple vmstarts)
+    res_over_run = all_res[run_key]
+    grouped = {}
+    for bench_key, timings in res_over_run.items():
+        for t in timings:
+            per_bench = grouped.setdefault(bench_key, {})
+            per_bench.setdefault(t['threads'], []).extend(t['warmiters'])
 
+    return grouped
 
-def main(argv):
-    results_file = argv[0]
-    assert os.path.exists(results_file)
-
-    with open(results_file, 'r') as f:
-        results = json.loads(f.read())
-
+def retrieve_data(results):
     print "RUNS:", len(results)
+    all_res = {}
     for run_key, run in results.iteritems():
         res_over_run = {}
 
@@ -38,19 +43,122 @@
                     print "############# THERE ARE FAILURES! #############"
                     #print "fail reasons:", failures
                     #import pdb;pdb.set_trace()
+        # print ""
+        # print "RESULTS OF RUN:"
+        all_res[run_key] = res_over_run
+        # grouped = collect_warmiters(all_res, run_key)
+        # for b, per_bench in grouped.items():
+        #     print "BENCH", b
+        #     for ts in sorted(per_bench.keys()):
+        #         if per_bench[ts]:
+        #             print "TS:", ts, "times:", per_bench[ts]
         print ""
-        print "RESULTS OF RUN:"
-        for bench_key, timings in res_over_run.items():
-            print "BENCH", bench_key
-            # group timings by thread (multiple vmstarts)
-            threads = results[run_key]['config']['defaults']['threads']
-            grouped = [[] for _ in range(max(threads))]
-            for t in timings:
-                grouped[t['threads'] - 1].extend(t['warmiters'])
+        print ""
+    return all_res
+
+
+def print_csv(all_res, run_key):
+    import numpy as np
+
+    grouped = collect_warmiters(all_res, run_key)
+    cols = len(grouped) * 2 + 1 # "threads" + avg, stddev for each benchmark
+    rows = len(grouped.values()[0]) + 1 # name + threads
+    table = [["" for _ in range(cols)] for _ in range(rows)] # t[row][col]
+
+    table[0][0] = "Threads"
+    for bench_num, (b, per_bench) in enumerate(grouped.items()):
+        print "BENCH", b
+        table[0][1 + bench_num * 2] = b
+        ts_index = 0
+        for ts in sorted(per_bench.keys()):
+            if per_bench[ts]:
+                row = 1 + ts_index
+                if table[row][0] == "":
+                    table[row][0] = ts
+                else:
+                    assert table[row][0] == ts
+
+                print "TS:", ts, "times:", per_bench[ts]
+                col = 1 + bench_num * 2
+                table[row][col] = np.mean(per_bench[ts])
+                table[row][col+1] = np.std(per_bench[ts])
+                ts_index += 1
+
+    # print table:
+    for r in range(rows):
+        line = ",\t".join(map(str, table[r]))
+        print line
+
+def print_latex_table(all_res, gil_run_key, stm_run_key):
+    print ""
+    print r"\footnotesize"
+    print r"\begin{tabularx}{\textwidth}{l|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r@{\hspace{5pt}}r@{\hspace{5pt}}r@{\hspace{5pt}}r|r}"
+    #print r"\hline"
+    print r"\textbf{Python VM} & \multicolumn{4}{c|}{\textbf{PyPy-GIL}} & \multicolumn{4}{c}{\textbf{PyPy-STM}} & \multicolumn{1}{|p{1cm}}{\textbf{Max. speedup}} \\ \hline"
+    print r"\textbf{Threads} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c|}{\textbf{8}} & \multicolumn{1}{c}{\textbf{1}} & \multicolumn{1}{c}{\textbf{2}} & \multicolumn{1}{c}{\textbf{4}} & \multicolumn{1}{c}{\textbf{8}} & \multicolumn{1}{|c}{*} \\ \hline"
+
+    gil_grouped = collect_warmiters(all_res, gil_run_key)
+    stm_grouped = collect_warmiters(all_res, stm_run_key)
+
+    assert stm_grouped.keys() == gil_grouped.keys()
+    warnings = ""
+    for bench_key in stm_grouped.keys():
+        elems = []
+        gil_bench = gil_grouped[bench_key]
+        stm_bench = stm_grouped[bench_key]
+        threads = [1, 2, 4, 8]
+        min_gil, min_stm = 9999999., 9999999.
+        for vm_bench in [gil_bench, stm_bench]:
             for ts in threads:
-                print "TS:", ts, "times:", grouped[ts - 1]
-        print ""
-        print ""
+                samples = vm_bench[ts]
+                if len(samples) < 30:
+                    warnings += "WARNING, %s had only %s samples\n" % (bench_key, len(samples))
+                mean, std = np.mean(samples), np.std(samples)
+                if vm_bench is gil_bench:
+                    if mean < min_gil:
+                        min_gil = mean
+                else:
+                    if mean < min_stm:
+                        min_stm = mean
+                elems.append((mean, std))
+
+        cells = []
+        min_mean = min(min_gil, min_stm)
+        for e in elems:
+            if e[0] == min_gil or e[0] == min_stm:
+                s = r"$\mathbf{%.2f}$ \scriptsize $\pm %.1f$ \footnotesize" % e
+            else:
+                s = r"$%.2f$ \scriptsize $\pm %.1f$ \footnotesize" % e
+            cells.append(s)
+        #
+        speedup = min_gil / min_stm
+        cells.append(r"$%.2f\times$" % speedup)
+        print r"%s & " % bench_key + " & ".join(cells) + r" \\"
+    print r"\hline"
+    print r"\end{tabularx}"
+    print r"\normalsize"
+    print ""
+    print warnings
+
+def main(argv):
+    results_file = argv[0]
+    assert os.path.exists(results_file)
+
+    with open(results_file, 'r') as f:
+        results = json.loads(f.read())
+
+    all_res = retrieve_data(results)
+    while True:
+        print "select gil and stm run (e.g., 0,1):"
+        runs = all_res.keys()
+        choices = ["%s: %s (%s)" % (i, r, results[r]['python'])
+                   for i, r in enumerate(runs)]
+        print "\n".join(choices)
+        choice = raw_input()
+
+        gil_run_key, stm_run_key = [runs[int(c)] for c in choice.split(',')]
+        print_csv(all_res, gil_run_key)
+        print_latex_table(all_res, gil_run_key, stm_run_key)
 
 
 if __name__ == '__main__':
diff --git a/multithread/runner.py b/multithread/runner.py
--- a/multithread/runner.py
+++ b/multithread/runner.py
@@ -9,6 +9,7 @@
 import pprint
 from subprocess import Popen, PIPE
 
+WITH_NUMACTL = True
 
 def extract_iter_times(stdout):
     for line in stdout.split('\n'):
@@ -35,6 +36,9 @@
                     str(bench_config['warmiters']),
                     str(ts)]
                    + bench_config['args'])
+            if WITH_NUMACTL:
+                # run on node 2, allocate preferrably on node 2
+                cmd = ["numactl", "-N2", "--preferred=2"] + cmd
             cmd_str = " ".join(cmd)
 
             cwd, _ = os.path.split(bench_file)

From pypy.commits at gmail.com  Mon Jun 20 09:46:41 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 20 Jun 2016 06:46:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: refactoring: split up
 NotVirtualStateInfo into the int version and the generic
Message-ID: <5767f3c1.cf981c0a.df69e.ffffd8be@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r85245:d59182c63df0
Date: 2016-06-20 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/d59182c63df0/

Log:	refactoring: split up NotVirtualStateInfo into the int version and
	the generic one (to be refined further)

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState,\
+     VArrayStateInfo, not_virtual, VirtualState,\
      GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
@@ -31,10 +31,10 @@
     def setup_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info = not_virtual(self.cpu, 'r', value)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info2 = not_virtual(self.cpu, 'r', value)
     
     def guards(self, info1, info2, box, runtime_box, expected, inputargs=None):
         if inputargs is None:
@@ -80,7 +80,7 @@
     def test_make_inputargs(self):
         optimizer = FakeOptimizer(self.cpu)
         args = [InputArgInt()]
-        info0 = NotVirtualStateInfo(self.cpu, args[0].type, None)
+        info0 = not_virtual(self.cpu, args[0].type, None)
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
@@ -108,8 +108,8 @@
             assert info1 in state.bad and info2 in state.bad
 
         for BoxType in (InputArgInt, InputArgFloat, InputArgRef):
-            info1 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
-            info2 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
+            info1 = not_virtual(self.cpu, BoxType.type, None)
+            info2 = not_virtual(self.cpu, BoxType.type, None)
             postest(info1, info2)
 
         info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42)
@@ -126,9 +126,9 @@
 
     def test_NotVirtualStateInfo_generalization(self):
         def isgeneral(tp1, info1, tp2, info2):
-            info1 = NotVirtualStateInfo(self.cpu, tp1, info1)
+            info1 = not_virtual(self.cpu, tp1, info1)
             info1.position = 0
-            info2 = NotVirtualStateInfo(self.cpu, tp2, info2)
+            info2 = not_virtual(self.cpu, tp2, info2)
             info2.position = 0
             return VirtualState([info1]).generalization_of(VirtualState([info2]), FakeOptimizer(self.cpu))
 
@@ -166,8 +166,8 @@
         assert not isgeneral('r', value1, 'r', value2)
 
     def test_field_matching_generalization(self):
-        const1 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(1))
-        const2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(2))
+        const1 = not_virtual(self.cpu, 'i', ConstIntBound(1))
+        const2 = not_virtual(self.cpu, 'i', ConstIntBound(2))
         const1.position = const2.position = 1
         self.check_invalid(const1, const2)
         self.check_invalid(const2, const1)
@@ -192,16 +192,16 @@
 
     def test_known_class_generalization(self):
         knownclass1 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info1 = NotVirtualStateInfo(self.cpu, 'r', knownclass1)
+        info1 = not_virtual(self.cpu, 'r', knownclass1)
         info1.position = 0
         knownclass2 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info2 = NotVirtualStateInfo(self.cpu, 'r', knownclass2)
+        info2 = not_virtual(self.cpu, 'r', knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
         knownclass3 = info.InstancePtrInfo(None, ConstPtr(self.myptr2))
-        info3 = NotVirtualStateInfo(self.cpu, 'r', knownclass3)
+        info3 = not_virtual(self.cpu, 'r', knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
         self.check_invalid(info2, info3)
@@ -222,26 +222,26 @@
         #unknown_val = PtrOptValue(self.nodebox)
         #unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
         opt = FakeOptimizer(self.cpu)
-        unknown_info = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info = not_virtual(self.cpu, 'r', None)
 
-        nonnull_info = NotVirtualStateInfo(self.cpu, 'r', info.NonNullPtrInfo())
+        nonnull_info = not_virtual(self.cpu, 'r', info.NonNullPtrInfo())
 
         classbox1 = self.cpu.ts.cls_of_box(ConstPtr(self.nodeaddr))
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox1))
+        knownclass_info = not_virtual(self.cpu, 'r',
+                                      info.InstancePtrInfo(None, classbox1))
         classbox2 = self.cpu.ts.cls_of_box(ConstPtr(self.node2addr))
-        knownclass2_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox2))
+        knownclass2_info = not_virtual(self.cpu, 'r',
+                                       info.InstancePtrInfo(None, classbox2))
 
-        constant_info = NotVirtualStateInfo(self.cpu, 'i',
-                                            ConstIntBound(1))
-        constant_ptr_info = NotVirtualStateInfo(self.cpu, 'r',
+        constant_info = not_virtual(self.cpu, 'i',
+                                    ConstIntBound(1))
+        constant_ptr_info = not_virtual(self.cpu, 'r',
                                     info.ConstPtrInfo(ConstPtr(self.nodeaddr)))
         constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr))
-        constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val)
-        constclass2_info = NotVirtualStateInfo(self.cpu, 'r',
+        constclass_info = not_virtual(self.cpu, 'r', constclass_val)
+        constclass2_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.node2addr)))
-        constantnull_info = NotVirtualStateInfo(self.cpu, 'r',
+        constantnull_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.nullptr)))
 
         # unknown unknown
@@ -392,8 +392,8 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', IntUnbounded())
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', IntUnbounded())
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -408,18 +408,18 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(10000))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(10000))
         self.check_invalid(info1, info2)
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(11))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(11))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value1 = info.InstancePtrInfo(None, classbox)
-        info1 = NotVirtualStateInfo(self.cpu, 'r', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        info1 = not_virtual(self.cpu, 'r', value1)
+        info2 = not_virtual(self.cpu, 'r', None)
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []        
@@ -456,18 +456,18 @@
     def test_equal_inputargs(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
+        unknown_info2 = not_virtual(self.cpu, 'r', None)
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate3.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -494,9 +494,9 @@
     def test_generate_guards_on_virtual_fields_matches_array(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         descr = ArrayDescr(lltype.GcArray(llmemory.GCREF), self.cpu)
@@ -524,9 +524,9 @@
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
@@ -552,9 +552,9 @@
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         structdescr = self.nodesize
@@ -583,9 +583,9 @@
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         NODE = lltype.Struct('NODE', ('x', llmemory.GCREF))
@@ -627,7 +627,7 @@
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info2.fieldstate = [unknown_info1, unknown_info1]
         vstate2 = VirtualState([info2])
@@ -636,9 +636,9 @@
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info3 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info2 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info3.fieldstate = [unknown_info1, unknown_info2]
         vstate3 = VirtualState([info3])        
@@ -651,7 +651,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -659,7 +659,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -671,7 +671,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -679,7 +679,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -691,7 +691,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -699,7 +699,7 @@
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -711,12 +711,12 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        info2 = not_virtual(self.cpu, 'r', value)
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
@@ -727,7 +727,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -735,7 +735,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -747,7 +747,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -755,7 +755,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -793,7 +793,7 @@
     def test_crash_varay_clear(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
         innerinfo1.position_in_notvirtuals = 0
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -350,21 +350,25 @@
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
 
+
+def not_virtual(cpu, type, info):
+    if type == 'i':
+        return NotVirtualStateInfoInt(cpu, type, info)
+    return NotVirtualStateInfo(cpu, type, info)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     lenbound = None
-    intbound = None
     level = LEVEL_UNKNOWN
     constbox = None
     known_class = None
-    
+
     def __init__(self, cpu, type, info):
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
             if type == 'r':
                 self.known_class = info.get_known_class(cpu)
-            elif type == 'i':
-                self.intbound = info
         elif type == 'r':
             if info:
                 self.known_class = info.get_known_class(cpu)
@@ -373,13 +377,6 @@
                 elif info.is_nonnull():
                     self.level = LEVEL_NONNULL
                 self.lenbound = info.getlenbound(None)
-        elif type == 'i':
-            if isinstance(info, IntBound):
-                if info.lower < MININT / 2:
-                    info.lower = MININT
-                if info.upper > MAXINT / 2:
-                    info.upper = MAXINT
-                self.intbound = info
         elif type == 'f':
             if info and info.is_constant():
                 self.level = LEVEL_CONSTANT
@@ -412,11 +409,9 @@
                 raise VirtualStatesCantMatch("length bound does not match")
 
         if self.level == LEVEL_UNKNOWN:
-            # confusingly enough, this is done also for pointers
-            # which have the full range as the "bound", so it always works
-            return self._generate_guards_intbounds(other, box, runtime_box,
-                                                   extra_guards,
-                                                   state.optimizer)
+            return self._generate_guards_unkown(other, box, runtime_box,
+                                                extra_guards,
+                                                state.optimizer)
 
         # the following conditions often peek into the runtime value that the
         # box had when tracing. This value is only used as an educated guess.
@@ -485,19 +480,9 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, box, runtime_box, extra_guards,
-                                   optimizer):
-        if self.intbound is None:
-            return
-        if self.intbound.contains_bound(other.intbound):
-            return
-        if (runtime_box is not None and
-            self.intbound.contains(runtime_box.getint())):
-            # this may generate a few more guards than needed, but they are
-            # optimized away when emitting them
-            self.intbound.make_guards(box, extra_guards, optimizer)
-            return
-        raise VirtualStatesCantMatch("intbounds don't match")
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                optimizer):
+        return
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         if self.level == LEVEL_CONSTANT:
@@ -553,8 +538,46 @@
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
 
-        debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
-                    ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
+        result = indent + mark + 'NotVirtualStateInfo(%d' % self.position + ', ' + l
+        extra = self._extra_repr()
+        if extra:
+            result += ', ' + extra
+        result += lb + ')'
+        debug_print(result)
+
+class NotVirtualStateInfoInt(NotVirtualStateInfo):
+    intbound = None
+
+    def __init__(self, cpu, type, info):
+        NotVirtualStateInfo.__init__(self, cpu, type, info)
+        assert type == 'i'
+        if isinstance(info, IntBound):
+            if info.lower < MININT / 2:
+                info.lower = MININT
+            if info.upper > MAXINT / 2:
+                info.upper = MAXINT
+            self.intbound = info
+
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                optimizer):
+        other_intbound = None
+        if isinstance(other, NotVirtualStateInfoInt):
+            other_intbound = other.intbound
+        if self.intbound is None:
+            return
+        if self.intbound.contains_bound(other_intbound):
+            return
+        if (runtime_box is not None and
+            self.intbound.contains(runtime_box.getint())):
+            # this may generate a few more guards than needed, but they are
+            # optimized away when emitting them
+            self.intbound.make_guards(box, extra_guards, optimizer)
+            return
+        raise VirtualStatesCantMatch("intbounds don't match")
+
+    def _extra_repr(self):
+        return self.intbound.__repr__()
+
 
 
 class VirtualState(object):
@@ -678,8 +701,8 @@
         return VirtualState(state)
 
     def visit_not_virtual(self, box):
-        return NotVirtualStateInfo(self.optimizer.cpu, box.type,
-                                   self.optimizer.getinfo(box))
+        return not_virtual(self.optimizer.cpu, box.type,
+                           self.optimizer.getinfo(box))
 
     def visit_virtual(self, descr, fielddescrs):
         known_class = ConstInt(descr.get_vtable())

From pypy.commits at gmail.com  Mon Jun 20 09:46:42 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 20 Jun 2016 06:46:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: move a lot of code into
 NotVirtualStateInfoPtr, that was so far applied to ints
Message-ID: <5767f3c2.c445c20a.9300e.fffff18d@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r85246:42550fb84ea8
Date: 2016-06-20 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/42550fb84ea8/

Log:	move a lot of code into NotVirtualStateInfoPtr, that was so far
	applied to ints and floats as well

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -260,9 +260,10 @@
         self.check_no_guards(unknown_info, knownclass_info)
 
         # unknown constant
-        self.check_no_guards(unknown_info, constant_info,
+        unknown_info_int = not_virtual(self.cpu, 'i', None)
+        self.check_no_guards(unknown_info_int, constant_info,
                              ConstInt(1), ConstIntBound(1))
-        self.check_no_guards(unknown_info, constant_info)
+        self.check_no_guards(unknown_info_int, constant_info)
 
 
         # nonnull unknown
@@ -293,11 +294,11 @@
         const_nonnull = ConstPtr(self.nodeaddr)
         const_nonnull2 = ConstPtr(self.node2addr)
         const_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
-        self.check_no_guards(nonnull_info, constant_info, const_nonnull,
+        self.check_no_guards(nonnull_info, constant_ptr_info, const_nonnull,
                              info.ConstPtrInfo(const_nonnull))
         self.check_invalid(nonnull_info, constantnull_info, const_null,
                            info.ConstPtrInfo(const_null))
-        self.check_no_guards(nonnull_info, constant_info)
+        self.check_no_guards(nonnull_info, constant_ptr_info)
         self.check_invalid(nonnull_info, constantnull_info)
 
 
@@ -706,7 +707,7 @@
 
         assert not vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
-        
+
     def test_nonvirtual_is_not_virtual(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -354,33 +354,19 @@
 def not_virtual(cpu, type, info):
     if type == 'i':
         return NotVirtualStateInfoInt(cpu, type, info)
+    if type == 'r':
+        return NotVirtualStateInfoPtr(cpu, type, info)
     return NotVirtualStateInfo(cpu, type, info)
 
 
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
-    lenbound = None
     level = LEVEL_UNKNOWN
     constbox = None
-    known_class = None
 
     def __init__(self, cpu, type, info):
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
-            if type == 'r':
-                self.known_class = info.get_known_class(cpu)
-        elif type == 'r':
-            if info:
-                self.known_class = info.get_known_class(cpu)
-                if self.known_class:
-                    self.level = LEVEL_KNOWNCLASS
-                elif info.is_nonnull():
-                    self.level = LEVEL_NONNULL
-                self.lenbound = info.getlenbound(None)
-        elif type == 'f':
-            if info and info.is_constant():
-                self.level = LEVEL_CONSTANT
-                self.constbox = info.getconst()
 
     def is_const(self):
         return self.constbox is not None
@@ -391,82 +377,15 @@
     def _generate_guards(self, other, box, runtime_box, state):
         # XXX This will always retrace instead of forcing anything which
         # might be what we want sometimes?
-        if not isinstance(other, NotVirtualStateInfo):
-            raise VirtualStatesCantMatch(
-                    'The VirtualStates does not match as a ' +
-                    'virtual appears where a pointer is needed ' +
-                    'and it is too late to force it.')
-
-
         extra_guards = state.extra_guards
-        cpu = state.cpu
-        if self.lenbound:
-            if other.lenbound is None:
-                other_bound = IntLowerBound(0)
-            else:
-                other_bound = other.lenbound
-            if not self.lenbound.contains_bound(other_bound):
-                raise VirtualStatesCantMatch("length bound does not match")
-
         if self.level == LEVEL_UNKNOWN:
             return self._generate_guards_unkown(other, box, runtime_box,
                                                 extra_guards,
-                                                state.optimizer)
-
-        # the following conditions often peek into the runtime value that the
-        # box had when tracing. This value is only used as an educated guess.
-        # It is used here to choose between either emitting a guard and jumping
-        # to an existing compiled loop or retracing the loop. Both alternatives
-        # will always generate correct behaviour, but performance will differ.
-        elif self.level == LEVEL_NONNULL:
-            if other.level == LEVEL_UNKNOWN:
-                if runtime_box is not None and runtime_box.nonnull():
-                    op = ResOperation(rop.GUARD_NONNULL, [box])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other not known to be nonnull")
-            elif other.level == LEVEL_NONNULL:
-                return
-            elif other.level == LEVEL_KNOWNCLASS:
-                return # implies nonnull
-            else:
-                assert other.level == LEVEL_CONSTANT
-                assert other.constbox
-                if not other.constbox.nonnull():
-                    raise VirtualStatesCantMatch("constant is null")
-                return
-
-        elif self.level == LEVEL_KNOWNCLASS:
-            if other.level == LEVEL_UNKNOWN:
-                if (runtime_box and runtime_box.nonnull() and
-              self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_NONNULL:
-                if (runtime_box and self.known_class.same_constant(
-                        cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_KNOWNCLASS:
-                if self.known_class.same_constant(other.known_class):
-                    return
-                raise VirtualStatesCantMatch("classes don't match")
-            else:
-                assert other.level == LEVEL_CONSTANT
-                if (other.constbox.nonnull() and
-                        self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))):
-                    return
-                else:
-                    raise VirtualStatesCantMatch("classes don't match")
-
+                                                state)
         else:
+            if not isinstance(other, NotVirtualStateInfo):
+                raise VirtualStatesCantMatch(
+                        'comparing a constant against something that is a virtual')
             assert self.level == LEVEL_CONSTANT
             if other.level == LEVEL_CONSTANT:
                 if self.constbox.same_constant(other.constbox):
@@ -481,7 +400,7 @@
         assert 0, "unreachable"
 
     def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
-                                optimizer):
+                                state):
         return
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
@@ -579,6 +498,105 @@
         return self.intbound.__repr__()
 
 
+class NotVirtualStateInfoPtr(NotVirtualStateInfo):
+    lenbound = None
+    known_class = None
+
+    def __init__(self, cpu, type, info):
+        if info:
+            self.known_class = info.get_known_class(cpu)
+            if self.known_class:
+                self.level = LEVEL_KNOWNCLASS
+            elif info.is_nonnull():
+                self.level = LEVEL_NONNULL
+            self.lenbound = info.getlenbound(None)
+        # might set it to LEVEL_CONSTANT
+        NotVirtualStateInfo.__init__(self, cpu, type, info)
+
+    def _generate_guards(self, other, box, runtime_box, state):
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch(
+                    'The VirtualStates does not match as a ' +
+                    'virtual appears where a pointer is needed ' +
+                    'and it is too late to force it.')
+        extra_guards = state.extra_guards
+        if self.lenbound:
+            if other.lenbound is None:
+                other_bound = IntLowerBound(0)
+            else:
+                other_bound = other.lenbound
+            if not self.lenbound.contains_bound(other_bound):
+                raise VirtualStatesCantMatch("length bound does not match")
+        if self.level == LEVEL_NONNULL:
+            return self._generate_guards_nonnull(other, box, runtime_box,
+                                                 extra_guards,
+                                                 state)
+        elif self.level == LEVEL_KNOWNCLASS:
+            return self._generate_guards_knownclass(other, box, runtime_box,
+                                                    extra_guards,
+                                                    state)
+        return NotVirtualStateInfo._generate_guards(self, other, box,
+                                                    runtime_box, state)
+
+
+    # the following methods often peek into the runtime value that the
+    # box had when tracing. This value is only used as an educated guess.
+    # It is used here to choose between either emitting a guard and jumping
+    # to an existing compiled loop or retracing the loop. Both alternatives
+    # will always generate correct behaviour, but performance will differ.
+
+    def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
+                                 state):
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!')
+        if other.level == LEVEL_UNKNOWN:
+            if runtime_box is not None and runtime_box.nonnull():
+                op = ResOperation(rop.GUARD_NONNULL, [box])
+                extra_guards.append(op)
+                return
+            else:
+                raise VirtualStatesCantMatch("other not known to be nonnull")
+        elif other.level == LEVEL_NONNULL:
+            pass
+        elif other.level == LEVEL_KNOWNCLASS:
+            pass # implies nonnull
+        else:
+            assert other.level == LEVEL_CONSTANT
+            assert other.constbox
+            if not other.constbox.nonnull():
+                raise VirtualStatesCantMatch("constant is null")
+
+    def _generate_guards_knownclass(self, other, box, runtime_box, extra_guards,
+                                    state):
+        cpu = state.cpu
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!')
+        if other.level == LEVEL_UNKNOWN:
+            if (runtime_box and runtime_box.nonnull() and
+                  self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
+                op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
+                extra_guards.append(op)
+            else:
+                raise VirtualStatesCantMatch("other's class is unknown")
+        elif other.level == LEVEL_NONNULL:
+            if (runtime_box and self.known_class.same_constant(
+                    cpu.ts.cls_of_box(runtime_box))):
+                op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
+                extra_guards.append(op)
+            else:
+                raise VirtualStatesCantMatch("other's class is unknown")
+        elif other.level == LEVEL_KNOWNCLASS:
+            if self.known_class.same_constant(other.known_class):
+                return
+            raise VirtualStatesCantMatch("classes don't match")
+        else:
+            assert other.level == LEVEL_CONSTANT
+            if (other.constbox.nonnull() and
+                    self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))):
+                pass
+            else:
+                raise VirtualStatesCantMatch("classes don't match")
+
 
 class VirtualState(object):
     def __init__(self, state):

From pypy.commits at gmail.com  Mon Jun 20 10:33:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 07:33:31 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.7: Doc tweaks
Message-ID: <5767febb.50991c0a.9b0b0.ffffe8ba@mx.google.com>

Author: Armin Rigo 
Branch: release-1.7
Changeset: r2719:52f417ae8d67
Date: 2016-06-20 16:17 +0200
http://bitbucket.org/cffi/cffi/changeset/52f417ae8d67/

Log:	Doc tweaks

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -636,6 +636,15 @@
 variables defined as dynamic macros (``#define myvar  (*fetchme())``).
 Before version 1.2, you need to write getter/setter functions.
 
+Note that if you declare a variable in ``cdef()`` without using
+``const``, CFFI assumes it is a read-write variable and generates two
+pieces of code, one to read it and one to write it.  If the variable
+cannot in fact be written to in C code, for one reason or another, it
+will not compile.  In this case, you can declare it as a constant: for
+example, instead of ``foo_t *myglob;`` you would use ``foo_t *const
+myglob;``.  Note also that ``const foo_t *myglob;``  is a *variable;* it
+contains a variable pointer to a constant ``foo_t``.
+
 
 Debugging dlopen'ed C libraries
 -------------------------------
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -44,81 +44,14 @@
 arguments.  In the above example it would be ``b"world"`` and ``b"hi
 there, %s!\n"``.  In general it is ``somestring.encode(myencoding)``.
 
-*This example does not call any C compiler.*
+*This example does not call any C compiler.  It works in the so-called
+ABI mode, which means that it will crash if you call some function or
+access some fields of a structure that was slightly misdeclared in the
+cdef().*
 
-
-.. _out-of-line-abi-level:
-
-Out-of-line example (ABI level, out-of-line)
---------------------------------------------
-
-In a real program, you would not include the ``ffi.cdef()`` in your
-main program's modules.  Instead, you can rewrite it as follows.  It
-massively reduces the import times, because it is slow to parse a
-large C header.  It also allows you to do more detailed checkings
-during build-time without worrying about performance (e.g. calling
-``cdef()`` many times with small pieces of declarations, based
-on the version of libraries detected on the system).
-
-*This example does not call any C compiler.*
-
-.. code-block:: python
-
-    # file "simple_example_build.py"
-
-    # Note: we instantiate the same 'cffi.FFI' class as in the previous
-    # example, but call the result 'ffibuilder' now instead of 'ffi';
-    # this is to avoid confusion with the other 'ffi' object you get below
-
-    from cffi import FFI
-
-    ffibuilder = FFI()
-    ffibuilder.set_source("_simple_example", None)
-    ffibuilder.cdef("""
-        int printf(const char *format, ...);
-    """)
-
-    if __name__ == "__main__":
-        ffibuilder.compile(verbose=True)
-
-Running it once produces ``_simple_example.py``.  Your main program
-only imports this generated module, not ``simple_example_build.py``
-any more:
-
-.. code-block:: python
-
-    from _simple_example import ffi
-
-    lib = ffi.dlopen(None)      # Unix: open the standard C library
-    #import ctypes.util         # or, try this on Windows:
-    #lib = ffi.dlopen(ctypes.util.find_library("c"))
-
-    lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
-
-Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
-does not invoke any additional magic to locate the library: it must be
-a path name (with or without a directory), as required by the C
-``dlopen()`` or ``LoadLibrary()`` functions.  This means that
-``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
-In the latter case, you could replace it with
-``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
-recognized on Unix to open the standard C library.
-
-For distribution purposes, remember that there is a new
-``_simple_example.py`` file generated.  You can either include it
-statically within your project's source files, or, with Setuptools,
-you can say in the ``setup.py``:
-
-.. code-block:: python
-
-    from setuptools import setup
-
-    setup(
-        ...
-        setup_requires=["cffi>=1.0.0"],
-        cffi_modules=["simple_example_build.py:ffibuilder"],
-        install_requires=["cffi>=1.0.0"],
-    )
+If using a C compiler to install your module is an option, it is highly
+recommended to use the API mode described in the next paragraph.  (It is
+also a bit faster at runtime.)
 
 
 .. _out-of-line-api-level:
@@ -131,6 +64,10 @@
 
     # file "example_build.py"
 
+    # Note: we instantiate the same 'cffi.FFI' class as in the previous
+    # example, but call the result 'ffibuilder' now instead of 'ffi';
+    # this is to avoid confusion with the other 'ffi' object you get below
+
     from cffi import FFI
     ffibuilder = FFI()
 
@@ -230,11 +167,11 @@
 *This example does not call any C compiler.*
 
 This example also admits an out-of-line equivalent.  It is similar to
-`Out-of-line example (ABI level, out-of-line)`_ above, but without any
-call to ``ffi.dlopen()``.  In the main program, you write ``from
-_simple_example import ffi`` and then the same content as the in-line
-example above starting from the line ``image = ffi.new("pixel_t[]",
-800*600)``.
+`Real example (API level, out-of-line)`_ above, but passing ``None`` as
+the second argument to ``ffibuilder.set_source()``.  Then in the main
+program you write ``from _simple_example import ffi`` and then the same
+content as the in-line example above starting from the line ``image =
+ffi.new("pixel_t[]", 800*600)``.
 
 
 .. _performance:
@@ -288,6 +225,77 @@
 distributed in precompiled form like any other extension module.*
 
 
+.. _out-of-line-abi-level:
+
+Out-of-line, ABI level
+----------------------
+
+The out-of-line ABI mode is a mixture of the regular (API) out-of-line
+mode and the in-line ABI mode.  It lets you use the ABI mode, with its
+advantages (not requiring a C compiler) and problems (crashes more
+easily).
+
+This mixture mode lets you massively reduces the import times, because
+it is slow to parse a large C header.  It also allows you to do more
+detailed checkings during build-time without worrying about performance
+(e.g. calling ``cdef()`` many times with small pieces of declarations,
+based on the version of libraries detected on the system).
+
+.. code-block:: python
+
+    # file "simple_example_build.py"
+
+    from cffi import FFI
+
+    ffibuilder = FFI()
+    ffibuilder.set_source("_simple_example", None)
+    ffibuilder.cdef("""
+        int printf(const char *format, ...);
+    """)
+
+    if __name__ == "__main__":
+        ffibuilder.compile(verbose=True)
+
+Running it once produces ``_simple_example.py``.  Your main program
+only imports this generated module, not ``simple_example_build.py``
+any more:
+
+.. code-block:: python
+
+    from _simple_example import ffi
+
+    lib = ffi.dlopen(None)      # Unix: open the standard C library
+    #import ctypes.util         # or, try this on Windows:
+    #lib = ffi.dlopen(ctypes.util.find_library("c"))
+
+    lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
+
+Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
+does not invoke any additional magic to locate the library: it must be
+a path name (with or without a directory), as required by the C
+``dlopen()`` or ``LoadLibrary()`` functions.  This means that
+``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
+In the latter case, you could replace it with
+``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
+recognized on Unix to open the standard C library.
+
+For distribution purposes, remember that there is a new
+``_simple_example.py`` file generated.  You can either include it
+statically within your project's source files, or, with Setuptools,
+you can say in the ``setup.py``:
+
+.. code-block:: python
+
+    from setuptools import setup
+
+    setup(
+        ...
+        setup_requires=["cffi>=1.0.0"],
+        cffi_modules=["simple_example_build.py:ffibuilder"],
+        install_requires=["cffi>=1.0.0"],
+    )
+
+
 .. _embedding:
 
 Embedding
@@ -418,18 +426,18 @@
 functions.)
 
 The generated piece of C code should be the same independently on the
-platform on which you run it (or the Python version),
-so in simple cases you can directly
-distribute the pre-generated C code and treat it as a regular C
-extension module.  The special Setuptools lines in the `example
+platform on which you run it (or the Python version), so in simple cases
+you can directly distribute the pre-generated C code and treat it as a
+regular C extension module (which depends on the ``_cffi_backend``
+module, on CPython).  The special Setuptools lines in the `example
 above`__ are meant for the more complicated cases where we need to
 regenerate the C sources as well---e.g. because the Python script that
-regenerates this file will itself look around the system to know what
-it should include or not.
+regenerates this file will itself look around the system to know what it
+should include or not.
 
 .. __: real-example_
 
-Note that the "API level + in-line" mode combination is deprecated.
-It used to be done with ``lib = ffi.verify("C header")``.  The
-out-of-line variant with ``set_source("modname", "C header")`` is
+Note that the "API level + in-line" mode combination exists but is long
+deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
+The out-of-line variant with ``set_source("modname", "C header")`` is
 preferred.
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -537,7 +537,7 @@
             lib.event_cb_register(lib.my_event_callback, userdata)
 
         def process_event(self, evt):
-            ...
+            print "got event!"
 
     @ffi.def_extern()
     def my_event_callback(evt, userdata):
@@ -566,7 +566,7 @@
             lib.event_cb_register(ll_widget, lib.my_event_callback)
 
         def process_event(self, evt):
-            ...
+            print "got event!"
 
     @ffi.def_extern()
     def my_event_callback(ll_widget, evt):
@@ -789,7 +789,7 @@
             lib.register_stuff_with_callback_and_voidp_arg(my_global_callback, handle)
 
         def some_method(self, x):
-            ...
+            print "method called!"
 
 (See also the section about `extern "Python"`_ above, where the same
 general style is used.)
@@ -881,7 +881,7 @@
 
 ``__cdecl`` is supported but is always the default so it can be left
 out.  In the ``cdef()``, you can also use ``WINAPI`` as equivalent to
-``__stdcall``.  As mentioned above, it is not needed (but doesn't
+``__stdcall``.  As mentioned above, it is mostly not needed (but doesn't
 hurt) to say ``WINAPI`` or ``__stdcall`` when declaring a plain
 function in the ``cdef()``.  (The difference can still be seen if you
 take explicitly a pointer to this function with ``ffi.addressof()``,

From pypy.commits at gmail.com  Mon Jun 20 10:35:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 07:35:06 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.7: md5/sha
Message-ID: <5767ff1a.c91d1c0a.da1af.ffffeb14@mx.google.com>

Author: Armin Rigo 
Branch: release-1.7
Changeset: r2720:ac63374a298c
Date: 2016-06-20 16:36 +0200
http://bitbucket.org/cffi/cffi/changeset/ac63374a298c/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,9 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.7.0.tar.gz
 
-   - MD5: ...
+   - MD5: 34122a545060cee58bab88feab57006d
 
-   - SHA: ...
+   - SHA: d8033f34e17c0c51bb834b27f6e8c59fc24ae72c
+
+   - SHA256: 6ed5dd6afd8361f34819c68aaebf9e8fc12b5a5893f91f50c9e50c8886bb60df
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Mon Jun 20 10:35:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 07:35:07 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.7
Message-ID: <5767ff1b.82e01c0a.f2b9e.7df4@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r2721:338244197c0d
Date: 2016-06-20 16:36 +0200
http://bitbucket.org/cffi/cffi/changeset/338244197c0d/

Log:	hg merge release-1.7

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -636,6 +636,15 @@
 variables defined as dynamic macros (``#define myvar  (*fetchme())``).
 Before version 1.2, you need to write getter/setter functions.
 
+Note that if you declare a variable in ``cdef()`` without using
+``const``, CFFI assumes it is a read-write variable and generates two
+pieces of code, one to read it and one to write it.  If the variable
+cannot in fact be written to in C code, for one reason or another, it
+will not compile.  In this case, you can declare it as a constant: for
+example, instead of ``foo_t *myglob;`` you would use ``foo_t *const
+myglob;``.  Note also that ``const foo_t *myglob;``  is a *variable;* it
+contains a variable pointer to a constant ``foo_t``.
+
 
 Debugging dlopen'ed C libraries
 -------------------------------
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,9 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.7.0.tar.gz
 
-   - MD5: ...
+   - MD5: 34122a545060cee58bab88feab57006d
 
-   - SHA: ...
+   - SHA: d8033f34e17c0c51bb834b27f6e8c59fc24ae72c
+
+   - SHA256: 6ed5dd6afd8361f34819c68aaebf9e8fc12b5a5893f91f50c9e50c8886bb60df
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -44,81 +44,14 @@
 arguments.  In the above example it would be ``b"world"`` and ``b"hi
 there, %s!\n"``.  In general it is ``somestring.encode(myencoding)``.
 
-*This example does not call any C compiler.*
+*This example does not call any C compiler.  It works in the so-called
+ABI mode, which means that it will crash if you call some function or
+access some fields of a structure that was slightly misdeclared in the
+cdef().*
 
-
-.. _out-of-line-abi-level:
-
-Out-of-line example (ABI level, out-of-line)
---------------------------------------------
-
-In a real program, you would not include the ``ffi.cdef()`` in your
-main program's modules.  Instead, you can rewrite it as follows.  It
-massively reduces the import times, because it is slow to parse a
-large C header.  It also allows you to do more detailed checkings
-during build-time without worrying about performance (e.g. calling
-``cdef()`` many times with small pieces of declarations, based
-on the version of libraries detected on the system).
-
-*This example does not call any C compiler.*
-
-.. code-block:: python
-
-    # file "simple_example_build.py"
-
-    # Note: we instantiate the same 'cffi.FFI' class as in the previous
-    # example, but call the result 'ffibuilder' now instead of 'ffi';
-    # this is to avoid confusion with the other 'ffi' object you get below
-
-    from cffi import FFI
-
-    ffibuilder = FFI()
-    ffibuilder.set_source("_simple_example", None)
-    ffibuilder.cdef("""
-        int printf(const char *format, ...);
-    """)
-
-    if __name__ == "__main__":
-        ffibuilder.compile(verbose=True)
-
-Running it once produces ``_simple_example.py``.  Your main program
-only imports this generated module, not ``simple_example_build.py``
-any more:
-
-.. code-block:: python
-
-    from _simple_example import ffi
-
-    lib = ffi.dlopen(None)      # Unix: open the standard C library
-    #import ctypes.util         # or, try this on Windows:
-    #lib = ffi.dlopen(ctypes.util.find_library("c"))
-
-    lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
-
-Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
-does not invoke any additional magic to locate the library: it must be
-a path name (with or without a directory), as required by the C
-``dlopen()`` or ``LoadLibrary()`` functions.  This means that
-``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
-In the latter case, you could replace it with
-``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
-recognized on Unix to open the standard C library.
-
-For distribution purposes, remember that there is a new
-``_simple_example.py`` file generated.  You can either include it
-statically within your project's source files, or, with Setuptools,
-you can say in the ``setup.py``:
-
-.. code-block:: python
-
-    from setuptools import setup
-
-    setup(
-        ...
-        setup_requires=["cffi>=1.0.0"],
-        cffi_modules=["simple_example_build.py:ffibuilder"],
-        install_requires=["cffi>=1.0.0"],
-    )
+If using a C compiler to install your module is an option, it is highly
+recommended to use the API mode described in the next paragraph.  (It is
+also a bit faster at runtime.)
 
 
 .. _out-of-line-api-level:
@@ -131,6 +64,10 @@
 
     # file "example_build.py"
 
+    # Note: we instantiate the same 'cffi.FFI' class as in the previous
+    # example, but call the result 'ffibuilder' now instead of 'ffi';
+    # this is to avoid confusion with the other 'ffi' object you get below
+
     from cffi import FFI
     ffibuilder = FFI()
 
@@ -230,11 +167,11 @@
 *This example does not call any C compiler.*
 
 This example also admits an out-of-line equivalent.  It is similar to
-`Out-of-line example (ABI level, out-of-line)`_ above, but without any
-call to ``ffi.dlopen()``.  In the main program, you write ``from
-_simple_example import ffi`` and then the same content as the in-line
-example above starting from the line ``image = ffi.new("pixel_t[]",
-800*600)``.
+`Real example (API level, out-of-line)`_ above, but passing ``None`` as
+the second argument to ``ffibuilder.set_source()``.  Then in the main
+program you write ``from _simple_example import ffi`` and then the same
+content as the in-line example above starting from the line ``image =
+ffi.new("pixel_t[]", 800*600)``.
 
 
 .. _performance:
@@ -288,6 +225,77 @@
 distributed in precompiled form like any other extension module.*
 
 
+.. _out-of-line-abi-level:
+
+Out-of-line, ABI level
+----------------------
+
+The out-of-line ABI mode is a mixture of the regular (API) out-of-line
+mode and the in-line ABI mode.  It lets you use the ABI mode, with its
+advantages (not requiring a C compiler) and problems (crashes more
+easily).
+
+This mixture mode lets you massively reduces the import times, because
+it is slow to parse a large C header.  It also allows you to do more
+detailed checkings during build-time without worrying about performance
+(e.g. calling ``cdef()`` many times with small pieces of declarations,
+based on the version of libraries detected on the system).
+
+.. code-block:: python
+
+    # file "simple_example_build.py"
+
+    from cffi import FFI
+
+    ffibuilder = FFI()
+    ffibuilder.set_source("_simple_example", None)
+    ffibuilder.cdef("""
+        int printf(const char *format, ...);
+    """)
+
+    if __name__ == "__main__":
+        ffibuilder.compile(verbose=True)
+
+Running it once produces ``_simple_example.py``.  Your main program
+only imports this generated module, not ``simple_example_build.py``
+any more:
+
+.. code-block:: python
+
+    from _simple_example import ffi
+
+    lib = ffi.dlopen(None)      # Unix: open the standard C library
+    #import ctypes.util         # or, try this on Windows:
+    #lib = ffi.dlopen(ctypes.util.find_library("c"))
+
+    lib.printf(b"hi there, number %d\n", ffi.cast("int", 2))
+
+Note that this ``ffi.dlopen()``, unlike the one from in-line mode,
+does not invoke any additional magic to locate the library: it must be
+a path name (with or without a directory), as required by the C
+``dlopen()`` or ``LoadLibrary()`` functions.  This means that
+``ffi.dlopen("libfoo.so")`` is ok, but ``ffi.dlopen("foo")`` is not.
+In the latter case, you could replace it with
+``ffi.dlopen(ctypes.util.find_library("foo"))``.  Also, None is only
+recognized on Unix to open the standard C library.
+
+For distribution purposes, remember that there is a new
+``_simple_example.py`` file generated.  You can either include it
+statically within your project's source files, or, with Setuptools,
+you can say in the ``setup.py``:
+
+.. code-block:: python
+
+    from setuptools import setup
+
+    setup(
+        ...
+        setup_requires=["cffi>=1.0.0"],
+        cffi_modules=["simple_example_build.py:ffibuilder"],
+        install_requires=["cffi>=1.0.0"],
+    )
+
+
 .. _embedding:
 
 Embedding
@@ -418,18 +426,18 @@
 functions.)
 
 The generated piece of C code should be the same independently on the
-platform on which you run it (or the Python version),
-so in simple cases you can directly
-distribute the pre-generated C code and treat it as a regular C
-extension module.  The special Setuptools lines in the `example
+platform on which you run it (or the Python version), so in simple cases
+you can directly distribute the pre-generated C code and treat it as a
+regular C extension module (which depends on the ``_cffi_backend``
+module, on CPython).  The special Setuptools lines in the `example
 above`__ are meant for the more complicated cases where we need to
 regenerate the C sources as well---e.g. because the Python script that
-regenerates this file will itself look around the system to know what
-it should include or not.
+regenerates this file will itself look around the system to know what it
+should include or not.
 
 .. __: real-example_
 
-Note that the "API level + in-line" mode combination is deprecated.
-It used to be done with ``lib = ffi.verify("C header")``.  The
-out-of-line variant with ``set_source("modname", "C header")`` is
+Note that the "API level + in-line" mode combination exists but is long
+deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
+The out-of-line variant with ``set_source("modname", "C header")`` is
 preferred.
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -537,7 +537,7 @@
             lib.event_cb_register(lib.my_event_callback, userdata)
 
         def process_event(self, evt):
-            ...
+            print "got event!"
 
     @ffi.def_extern()
     def my_event_callback(evt, userdata):
@@ -566,7 +566,7 @@
             lib.event_cb_register(ll_widget, lib.my_event_callback)
 
         def process_event(self, evt):
-            ...
+            print "got event!"
 
     @ffi.def_extern()
     def my_event_callback(ll_widget, evt):
@@ -789,7 +789,7 @@
             lib.register_stuff_with_callback_and_voidp_arg(my_global_callback, handle)
 
         def some_method(self, x):
-            ...
+            print "method called!"
 
 (See also the section about `extern "Python"`_ above, where the same
 general style is used.)
@@ -881,7 +881,7 @@
 
 ``__cdecl`` is supported but is always the default so it can be left
 out.  In the ``cdef()``, you can also use ``WINAPI`` as equivalent to
-``__stdcall``.  As mentioned above, it is not needed (but doesn't
+``__stdcall``.  As mentioned above, it is mostly not needed (but doesn't
 hurt) to say ``WINAPI`` or ``__stdcall`` when declaring a plain
 function in the ``cdef()``.  (The difference can still be seen if you
 take explicitly a pointer to this function with ``ffi.addressof()``,

From pypy.commits at gmail.com  Mon Jun 20 11:44:35 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Jun 2016 08:44:35 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: unaligned load in software.
 there is no way around that for integer vector loads. store has some flaw,
 but it seems to work for integer add
Message-ID: <57680f63.c72d1c0a.5bbad.ffffffbf@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85247:21826e946450
Date: 2016-06-20 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/21826e946450/

Log:	unaligned load in software. there is no way around that for integer
	vector loads. store has some flaw, but it seems to work for integer
	add

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -63,7 +63,11 @@
 XLL = Form("LL", "XO1")
 XX1 = Form("fvrT", "rA", "rB", "XO1")
 XX3 = Form("fvrT", "fvrA", "fvrB", "XO9")
-VX = Form("lvrT", "lvrA", "lvrB", "XO8")
+XV = Form("ivrT", "rA", "rB", "XO1")
+VX = Form("ivrT", "ivrA", "ivrB", "XO8")
+VXI = Form("ivrT", "SIM", "XO8")
+VA = Form("ivrT", "ivrA", "ivrB", "ivrC", "XO10")
+
 
 MI = Form("rA", "rS", "SH", "MB", "ME", "Rc")
 MB = Form("rA", "rS", "rB", "MB", "ME", "Rc")
@@ -609,8 +613,27 @@
 
     # INTEGER
     # -------
+
+    # load & store
+    lvx = XV(31, XO1=103)
+    stvx = XV(31, XO1=231)
+
+    # arith & logic
     vaddudm = VX(4, XO8=192)
 
+    # shift, perm and select
+    lvsl = XV(31, XO1=6)
+    lvsr = XV(31, XO1=38)
+    vperm = VA(4, XO10=43)
+    vsel = VA(4, XO10=42) 
+    vspltisb = VXI(4, XO8=780)
+
+
+
+
+
+
+
 class PPCAssembler(BasicPPCAssembler, PPCVSXAssembler):
     BA = BasicPPCAssembler
 
diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py
--- a/rpython/jit/backend/ppc/ppc_field.py
+++ b/rpython/jit/backend/ppc/ppc_field.py
@@ -48,11 +48,12 @@
     "fvrB":   (16, 31, 'unsigned', regname._V, 'overlap'),
     # low vector register T (low in a sense:
     # can only address 32 vector registers)
-    "lvrT":   (6,  10, 'unsigned', regname._V),
+    "ivrT":   (6,  10, 'unsigned', regname._V),
     # low vector register A
-    "lvrA":   (11, 15, 'unsigned', regname._V),
+    "ivrA":   (11, 15, 'unsigned', regname._V),
     # low vector register B
-    "lvrB":   (16, 20, 'unsigned', regname._V),
+    "ivrB":   (16, 20, 'unsigned', regname._V),
+    "ivrC":   (21, 25, 'unsigned', regname._V),
     "XO1":    (21, 30),
     "XO2":    (22, 30),
     "XO3":    (26, 30),
@@ -62,7 +63,9 @@
     "XO7":    (27, 30),
     "XO8":    (21, 31),
     "XO9":    (21, 28),
+    "XO10":   (26, 31),
     "LL":     ( 9, 10),
+    "SIM":    (11, 15),
 }
 
 
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -10,7 +10,7 @@
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg, check_imm_box
 from rpython.jit.backend.ppc.helper import regalloc as helper
 from rpython.jit.metainterp.history import (Const, ConstInt, ConstFloat, ConstPtr,
-                                            INT, REF, FLOAT, VOID)
+                                            INT, REF, FLOAT, VOID, VECTOR)
 from rpython.jit.metainterp.history import JitCellToken, TargetToken
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.ppc import locations
@@ -50,6 +50,11 @@
     def __repr__(self):
         return "" % (id(self),)
 
+class TempVector(TempVar):
+    type = VECTOR
+
+    def __repr__(self):
+        return "" % (id(self),)
 
 class FPRegisterManager(RegisterManager):
     all_regs              = r.MANAGED_FP_REGS
@@ -136,9 +141,9 @@
         self.temp_boxes.append(box)
         return reg
 
-class VectorRegisterManager(RegisterManager):
-    all_regs              = r.MANAGED_VECTOR_REGS
-    box_types             = [INT, FLOAT]
+class IntegerVectorRegisterManager(RegisterManager):
+    all_regs              = r.MANAGED_INTEGER_VECTOR_REGS
+    box_types             = [INT]
     save_around_call_regs = [] # ??? lookup the ABI
     assert set(save_around_call_regs).issubset(all_regs)
 
@@ -148,6 +153,30 @@
     def ensure_reg(self, box):
         raise NotImplementedError
 
+    def get_scratch_reg(self):
+        box = TempInt()
+        reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes)
+        self.temp_boxes.append(box)
+        return reg
+
+class FloatVectorRegisterManager(IntegerVectorRegisterManager):
+    all_regs              = r.MANAGED_FLOAT_VECTOR_REGS
+    box_types             = [FLOAT]
+    save_around_call_regs = [] # ??? lookup the ABI
+    assert set(save_around_call_regs).issubset(all_regs)
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        RegisterManager.__init__(self, longevity, frame_manager, assembler)
+
+    def ensure_reg(self, box):
+        raise NotImplementedError
+
+    def get_scratch_reg(self):
+        box = TempFloat()
+        reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes)
+        self.temp_boxes.append(box)
+        return reg
+
 class PPCFrameManager(FrameManager):
     def __init__(self, base_ofs):
         FrameManager.__init__(self)
@@ -191,7 +220,9 @@
                                      assembler = self.assembler)
         self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm,
                                       assembler = self.assembler)
-        self.vrm = VectorRegisterManager(self.longevity, frame_manager = self.fm,
+        self.vrm = FloatVectorRegisterManager(self.longevity, frame_manager = self.fm,
+                                      assembler = self.assembler)
+        self.ivrm = IntegerVectorRegisterManager(self.longevity, frame_manager = self.fm,
                                       assembler = self.assembler)
         return operations
 
@@ -255,11 +286,15 @@
     def possibly_free_var(self, var):
         if var is not None:
             if var.type == FLOAT:
-                self.fprm.possibly_free_var(var)
-            elif var.is_vector() and var.type != VOID:
-                self.vrm.possibly_free_var(var)
-            else:
-                self.rm.possibly_free_var(var)
+                if var.is_vector():
+                    self.vrm.possibly_free_var(var)
+                else:
+                    self.fprm.possibly_free_var(var)
+            elif var.type == INT:
+                if var.is_vector():
+                    self.ivrm.possibly_free_var(var)
+                else:
+                    self.rm.possibly_free_var(var)
 
     def possibly_free_vars(self, vars):
         for var in vars:
@@ -303,6 +338,7 @@
             self.rm.position = i
             self.fprm.position = i
             self.vrm.position = i
+            self.ivrm.position = i
             opnum = op.opnum
             if rop.has_no_side_effect(opnum) and op not in self.longevity:
                 i += 1
@@ -311,12 +347,16 @@
             #
             for j in range(op.numargs()):
                 box = op.getarg(j)
-                if box.is_vector():
-                    self.vrm.temp_boxes.append(box)
-                elif box.type != FLOAT:
-                    self.rm.temp_boxes.append(box)
+                if box.type != FLOAT:
+                    if box.is_vector():
+                        self.ivrm.temp_boxes.append(box)
+                    else:
+                        self.rm.temp_boxes.append(box)
                 else:
-                    self.fprm.temp_boxes.append(box)
+                    if box.is_vector():
+                        self.vrm.temp_boxes.append(box)
+                    else:
+                        self.fprm.temp_boxes.append(box)
             #
             if not we_are_translated() and opnum == rop.FORCE_SPILL:
                 self._consider_force_spill(op)
@@ -328,6 +368,7 @@
             self.rm._check_invariants()
             self.fprm._check_invariants()
             self.vrm._check_invariants()
+            self.ivrm._check_invariants()
             if self.assembler.mc.get_relative_pos() > self.limit_loop_break:
                 self.assembler.break_long_loop()
                 self.limit_loop_break = (self.assembler.mc.get_relative_pos() +
@@ -439,6 +480,7 @@
         self.rm.free_temp_vars()
         self.fprm.free_temp_vars()
         self.vrm.free_temp_vars()
+        self.ivrm.free_temp_vars()
 
     # ******************************************************
     # *         P R E P A R E  O P E R A T I O N S         * 
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -3,7 +3,8 @@
 
 ALL_REGS        = [RegisterLocation(i) for i in range(32)]
 ALL_FLOAT_REGS  = [FPRegisterLocation(i) for i in range(32)]
-ALL_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(64)]
+ALL_INTEGER_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(32)]
+ALL_FLOAT_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(64)]
 
 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16,\
     r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31\
@@ -13,12 +14,18 @@
     f17, f18, f19, f20, f21, f22, f23, f24, f25, f26, f27, f28, f29, f30, f31\
     = ALL_FLOAT_REGS
 
+ivr0, ivr1, ivr2, ivr3, ivr4, ivr5, ivr6, ivr7, ivr8, ivr9, ivr10, ivr11, ivr12,\
+    ivr13, ivr14, ivr15, ivr16, ivr17, ivr18, ivr19, ivr20, ivr21, ivr22, ivr23,\
+    ivr24, ivr25, ivr26, ivr27, ivr28, ivr29, ivr30, ivr31\
+    = ALL_FLOAT_REGS
+
 vr0, vr1, vr2, vr3, vr4, vr5, vr6, vr7, vr8, vr9, vr10, vr11, vr12, vr13, \
      vr14, vr15, vr16, vr17, vr18, vr19, vr20, vr21, vr22, vr23, vr24, vr25, \
      vr26, vr27, vr28, vr29, vr30, vr31, vr32, vr33, vr34, vr35, vr36, vr37, \
      vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \
      vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \
-     vr61, vr62, vr63 = ALL_VECTOR_REGS
+     vr61, vr62, vr63 = ALL_FLOAT_VECTOR_REGS
+
 
 NONVOLATILES        = [r14, r15, r16, r17, r18, r19, r20, r21, r22, r23,
                     r24, r25, r26, r27, r28, r29, r30, r31]
@@ -51,7 +58,8 @@
 
 MANAGED_FP_REGS = VOLATILES_FLOAT #+ NONVOLATILES_FLOAT
 
-MANAGED_VECTOR_REGS = ALL_VECTOR_REGS
+MANAGED_FLOAT_VECTOR_REGS = ALL_FLOAT_VECTOR_REGS
+MANAGED_INTEGER_VECTOR_REGS = ALL_INTEGER_VECTOR_REGS
 
 assert RCS1 in MANAGED_REGS and RCS1 in NONVOLATILES
 assert RCS2 in MANAGED_REGS and RCS2 in NONVOLATILES
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -36,24 +36,30 @@
     emit_vec_getarrayitem_gc_i = _emit_getitem
     emit_vec_getarrayitem_gc_f = _emit_getitem
 
-    def _emit_load(self, op, arglocs, regalloc):
-        resloc, base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+    def emit_vec_raw_load_f(self, op, arglocs, regalloc):
+        resloc, baseloc, indexloc, size_loc, ofs, integer_loc, aligned_loc = arglocs
+        #src_addr = addr_add(baseloc, ofs_loc, ofs.value, 0)
+        assert ofs.value == 0
+        itemsize = size_loc.value
+        if itemsize == 4:
+            self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 8:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+
+    def emit_vec_raw_load_i(self, op, arglocs, regalloc):
+        resloc, baseloc, indexloc, size_loc, ofs, \
+            Vhiloc, Vloloc, Vploc, tloc = arglocs
         #src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
         assert ofs.value == 0
-        self._vec_load(resloc, base_loc, ofs_loc, integer_loc.value,
-                       size_loc.value, aligned_loc.value)
-
-    emit_vec_raw_load_i = _emit_load
-    emit_vec_raw_load_f = _emit_load
-
-    def _vec_load(self, resloc, baseloc, indexloc, integer, itemsize, aligned):
-        if integer:
-            raise NotImplementedError
-        else:
-            if itemsize == 4:
-                self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
-            elif itemsize == 8:
-                self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        Vlo = Vloloc.value
+        Vhi = Vhiloc.value
+        self.mc.lvx(Vhi, indexloc.value, baseloc.value)
+        Vp = Vploc.value
+        t = tloc.value
+        self.mc.lvsl(Vp, indexloc.value, baseloc.value)
+        self.mc.addi(t, baseloc.value, 16)
+        self.mc.lvx(Vlo, indexloc.value, t)
+        self.mc.vperm(resloc.value, Vhi, Vlo, Vp)
 
     def _emit_vec_setitem(self, op, arglocs, regalloc):
         # prepares item scale (raw_store does not)
@@ -72,11 +78,41 @@
         #dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
         assert baseofs.value == 0
         self._vec_store(baseloc, ofsloc, valueloc, integer_loc.value,
-                        size_loc.value, aligned_loc.value)
+                        size_loc.value, regalloc)
 
-    def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, aligned):
+    def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, regalloc):
         if integer:
-            raise NotImplementedError
+            Vloloc = regalloc.ivrm.get_scratch_reg()
+            Vhiloc = regalloc.ivrm.get_scratch_reg()
+            Vploc = regalloc.ivrm.get_scratch_reg()
+            tloc = regalloc.rm.get_scratch_reg()
+            V1sloc = regalloc.ivrm.get_scratch_reg()
+            V1s = V1sloc.value
+            V0sloc = regalloc.ivrm.get_scratch_reg()
+            V0s = V0sloc.value
+            Vmaskloc = regalloc.ivrm.get_scratch_reg()
+            Vmask = Vmaskloc.value
+            Vlo = Vhiloc.value
+            Vhi = Vloloc.value
+            Vp = Vploc.value
+            t = tloc.value
+            Vs = valueloc.value
+            # UFF, that is a lot of code for storing unaligned!
+            # probably a lot of room for improvement (not locally,
+            # but in general for the algorithm)
+            self.mc.lvx(Vhi, indexloc.value, baseloc.value)
+            self.mc.lvsr(Vp, indexloc.value, baseloc.value)
+            self.mc.addi(t, baseloc.value, 16)
+            self.mc.lvx(Vlo, indexloc.value, t)
+            self.mc.vspltisb(V1s, -1)
+            self.mc.vspltisb(V0s, 0)
+            self.mc.vperm(Vmask, V0s, V1s, Vp)
+            self.mc.vperm(Vs, Vs, Vs, Vp)
+            self.mc.vsel(Vlo, Vs, Vlo, Vmask)
+            self.mc.vsel(Vhi, Vhi, Vs, Vmask)
+            self.mc.stvx(Vlo, indexloc.value, baseloc.value)
+            self.mc.addi(t, baseloc.value, -16)
+            self.mc.stvx(Vhi, indexloc.value, t)
         else:
             if itemsize == 4:
                 self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
@@ -94,7 +130,6 @@
         elif size == 4:
             raise NotImplementedError
         elif size == 8:
-            raise NotImplementedError # need value in another register!
             self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
 
     def emit_vec_float_add(self, op, arglocs, resloc):
@@ -531,12 +566,18 @@
 
     def force_allocate_vector_reg(self, op):
         forbidden_vars = self.vrm.temp_boxes
-        return self.vrm.force_allocate_reg(op, forbidden_vars)
+        if op.type == FLOAT:
+            return self.vrm.force_allocate_reg(op, forbidden_vars)
+        else:
+            return self.ivrm.force_allocate_reg(op, forbidden_vars)
 
     def ensure_vector_reg(self, box):
-        loc = self.vrm.make_sure_var_in_reg(box,
-                           forbidden_vars=self.vrm.temp_boxes)
-        return loc
+        if box.type == FLOAT:
+            return self.vrm.make_sure_var_in_reg(box,
+                               forbidden_vars=self.vrm.temp_boxes)
+        else:
+            return self.ivrm.make_sure_var_in_reg(box,
+                               forbidden_vars=self.ivrm.temp_boxes)
 
     def _prepare_load(self, op):
         descr = op.getdescr()
@@ -555,11 +596,30 @@
         return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
                 imm(integer), imm(aligned)]
 
-    prepare_vec_getarrayitem_raw_i = _prepare_load
+    def _prepare_load_i(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        args = op.getarglist()
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        base_loc = self.ensure_reg(a0)
+        ofs_loc = self.ensure_reg(a1)
+        result_loc = self.force_allocate_vector_reg(op)
+        tloc = self.rm.get_scratch_reg()
+        Vhiloc = self.ivrm.get_scratch_reg()
+        Vloloc = self.ivrm.get_scratch_reg()
+        Vploc = self.ivrm.get_scratch_reg()
+        return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
+                Vhiloc, Vloloc, Vploc, tloc]
+
+    prepare_vec_getarrayitem_raw_i = _prepare_load_i
     prepare_vec_getarrayitem_raw_f = _prepare_load
-    prepare_vec_getarrayitem_gc_i = _prepare_load
+    prepare_vec_getarrayitem_gc_i = _prepare_load_i
     prepare_vec_getarrayitem_gc_f = _prepare_load
-    prepare_vec_raw_load_i = _prepare_load
+    prepare_vec_raw_load_i = _prepare_load_i
     prepare_vec_raw_load_f = _prepare_load
 
     def prepare_vec_arith(self, op):
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -61,6 +61,15 @@
 integers_64bit = st.integers(min_value=-2**63, max_value=2**63-1)
 floats = st.floats()
 
+def rdiv(v1,v2):
+    # TODO unused, interpeting this on top of llgraph does not work correctly
+    try:
+        return v1 / v2
+    except ZeroDivisionError:
+        if v1 == v2 == 0.0:
+            return rfloat.NAN
+        return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+
 class VectorizeTests:
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -75,19 +84,9 @@
                               type_system=self.type_system,
                               vec=vec, vec_all=vec_all)
 
-    @staticmethod
-    def rdiv(v1,v2):
-        try:
-            return v1 / v2
-        except ZeroDivisionError:
-            if v1 == v2 == 0.0:
-                return rfloat.NAN
-            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
-
     @given(data=st.data())
     @pytest.mark.parametrize('func', [lambda a,b: a+b,
-        lambda a,b: a*b, lambda a,b: a-b,
-        lambda a,b: VectorizeTests.rdiv(a,b)])
+        lambda a,b: a*b, lambda a,b: a-b])
     def test_vector_simple_float(self, func, data):
         func = always_inline(func)
 
@@ -144,7 +143,7 @@
 
         rawstorage = RawStorage()
         #la = data.draw(st.lists(integers_64bit, min_size=10, max_size=150))
-        la = [0] * 10
+        la = [1] * 10
         l = len(la)
         #lb = data.draw(st.lists(integers_64bit, min_size=l, max_size=l))
         lb = [0] * 10

From pypy.commits at gmail.com  Mon Jun 20 12:22:35 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 20 Jun 2016 09:22:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix bug if keyword.arg is None on keyword
 argument unpack, insert new pyopcodes in dispatch method
Message-ID: <5768184b.24f9c20a.1929.49c4@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85248:e451b4b82bb1
Date: 2016-06-20 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/e451b4b82bb1/

Log:	Fix bug if keyword.arg is None on keyword argument unpack, insert
	new pyopcodes in dispatch method

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -397,7 +397,6 @@
         for block in blocks:
             depth = self._do_stack_depth_walk(block)
             if block.auto_inserted_return and depth != 0:
-                #depth is 1, import pdb;pdb.pm()
                 os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
                     self.compile_info.filename, self.name, self.first_lineno))
                 raise StackDepthComputationError   # fatal error
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1128,7 +1128,8 @@
         self.name_op(name.id, name.ctx)
 
     def visit_keyword(self, keyword):
-        self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
+        if keyword.arg is not None:
+            self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
         keyword.value.walkabout(self)
 
     def _make_call(self, n, # args already pushed
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -258,6 +258,16 @@
                 self.BUILD_MAP(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_SET.index:
                 self.BUILD_SET(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_SET_UNPACK.index:
+                self.BUILD_SET_UNPACK(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_TUPLE_UNPACK.index:
+                self.BUILD_TUPLE_UNPACK(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_LIST_UNPACK.index:
+                self.BUILD_LIST_UNPACK(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_MAP_UNPACK.index:
+                self.BUILD_MAP_UNPACK(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_MAP_UNPACK_WITH_CALL.index:
+                self.BUILD_MAP_UNPACK_WITH_CALL(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_SLICE.index:
                 self.BUILD_SLICE(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_TUPLE.index:

From pypy.commits at gmail.com  Mon Jun 20 12:30:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 09:30:44 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: (ssbr,
 ronan) Prevent unused variable warning in test_no_structmember()
Message-ID: <57681a34.2457c20a.ec0a.2e77@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85249:7c4dd42a0e24
Date: 2016-06-20 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/7c4dd42a0e24/

Log:	(ssbr, ronan) Prevent unused variable warning in
	test_no_structmember()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -985,7 +985,7 @@
             ('bar', 'METH_NOARGS',
              '''
              /* reuse a name that is #defined in structmember.h */
-             int RO;
+             int RO = 0; (void)RO;
              Py_RETURN_NONE;
              '''
              ),

From pypy.commits at gmail.com  Mon Jun 20 12:49:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 09:49:42 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Clean up,
 document and test the exact order of the messages and
Message-ID: <57681ea6.cbc71c0a.f2f7b.283b@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85250:9c2a20771e65
Date: 2016-06-20 18:33 +0200
http://bitbucket.org/pypy/pypy/changeset/9c2a20771e65/

Log:	Clean up, document and test the exact order of the messages and use
	it to implement "first" or "last" breakpoints

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -57,13 +57,13 @@
     unique id greater or equal."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'u')
 
- at specialize.arg(1)
-def go_forward(time_delta, callback):
-    """For RPython debug commands: tells that after this function finishes,
-    the debugger should run the 'forward ' command and then
-    invoke the 'callback' with no argument.
-    """
-    _change_time('f', time_delta, callback)
+## @specialize.arg(1)
+## def go_forward(time_delta, callback):
+##     """For RPython debug commands: tells that after this function finishes,
+##     the debugger should run the 'forward ' command and then
+##     invoke the 'callback' with no argument.
+##     """
+##     _change_time('f', time_delta, callback)
 
 def breakpoint(num):
     llop.revdb_breakpoint(lltype.Void, num)
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -88,6 +88,23 @@
         lst = [str(n) for n in sorted(self.pgroup.paused)]
         print ', '.join(lst)
 
+    def move_forward(self, steps):
+        try:
+            self.pgroup.go_forward(steps)
+        except Breakpoint as b:
+            self.hit_breakpoint(b)
+
+    def move_backward(self, steps):
+        try:
+            self.pgroup.go_backward(steps)
+        except Breakpoint as b:
+            self.hit_breakpoint(b)
+
+    def hit_breakpoint(self, b):
+        print 'Hit breakpoint %d' % (b.num,)
+        if self.pgroup.get_current_time() != b.time:
+            self.pgroup.jump_in_time(b.time)
+
     def command_step(self, argument):
         """Run forward ARG steps (default 1)"""
         arg = int(argument or '1')
@@ -97,16 +114,10 @@
         self.move_forward(arg)
     command_s = command_step
 
-    def move_forward(self, steps):
-        try:
-            self.pgroup.go_forward(steps)
-        except Breakpoint as b:
-            print 'Hit breakpoint %d' % (b.num,)
-
     def command_bstep(self, argument):
         """Run backward ARG steps (default 1)"""
         arg = int(argument or '1')
-        self.pgroup.jump_in_time(self.pgroup.get_current_time() - arg)
+        self.move_backward(arg)
     command_bs = command_bstep
 
     def command_continue(self, argument):
@@ -115,6 +126,11 @@
                           self.pgroup.get_current_time())
     command_c = command_continue
 
+    def command_bcontinue(self, argument):
+        """Run backward"""
+        self.move_backward(self.pgroup.get_current_time() - 1)
+    command_bc = command_bcontinue
+
     def command_print(self, argument):
         """Print an expression"""
         self.pgroup.print_cmd(argument)
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -1,25 +1,54 @@
 
 INIT_VERSION_NUMBER = 0xd80100
 
-CMD_FORK     = -1
-CMD_QUIT     = -2
-CMD_FORWARD  = -3
+
+# See the corresponding answers for details about messages.
+
+CMD_FORK     = -1      # Message(CMD_FORK)
+CMD_QUIT     = -2      # Message(CMD_QUIT)
+CMD_FORWARD  = -3      # Message(CMD_FORWARD, steps, breakpoint_mode)
 # extra commands which are not handled by revdb.c, but
 # by revdb.register_debug_command()
-CMD_PRINT       = 1
-CMD_BACKTRACE   = 2
-CMD_LOCALS      = 3
-CMD_BREAKPOINTS = 4
+CMD_PRINT       = 1    # Message(CMD_PRINT, extra=expression)
+CMD_BACKTRACE   = 2    # Message(CMD_BACKTRACE)
+CMD_LOCALS      = 3    # Message(CMD_LOCALS)
+CMD_BREAKPOINTS = 4    # Message(CMD_BREAKPOINTS, extra="\0-separated names")
 
+
+
+# the first message sent by the first child:
+#    Message(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points)
 ANSWER_INIT       = -20
-ANSWER_STD        = -21
+
+# sent when the child is done and waiting for the next command:
+#    Message(ANSWER_READY, current_time, currently_created_objects)
+ANSWER_READY      = -21
+
+# sent after CMD_FORK:
+#    Message(ANSWER_FORKED, child_pid)
 ANSWER_FORKED     = -22
+
+# sent when a child reaches the end (should not occur with process.py)
+#    Message(ANSWER_AT_END)           (the child exits afterwards)
 ANSWER_AT_END     = -23
+
+# breakpoint detected in CMD_FORWARD:
+#    Message(ANSWER_BREAKPOINT, break_time, break_created_objects, bpkt_num)
+# if breakpoint_mode=='b': sent immediately when seeing a breakpoint,
+#                          followed by ANSWER_STD with the same time
+# if breakpoint_mode=='r': sent when we're done going forward, about
+#                          the most recently seen breakpoint
+# if breakpoint_mode=='i': ignored, never sent
 ANSWER_BREAKPOINT = -24
 
+# print text to the console, for CMD_PRINT and others
+#    Message(ANSWER_TEXT, extra=text)
 ANSWER_TEXT       = 20
 
 
+# ____________________________________________________________
+
+
 class Message(object):
     """Represent messages sent and received to subprocesses
     started with --revdb-replay.
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -1,5 +1,4 @@
 import sys, os, struct, socket, errno, subprocess
-from contextlib import contextmanager
 from rpython.translator.revdb import ancillary
 from rpython.translator.revdb.message import *
 
@@ -8,7 +7,8 @@
 
 
 class Breakpoint(Exception):
-    def __init__(self, num):
+    def __init__(self, time, num):
+        self.time = time
         self.num = num
 
 
@@ -58,8 +58,8 @@
             assert msg.extra == extra
         return msg
 
-    def expect_std(self):
-        msg = self.expect(ANSWER_STD, Ellipsis, Ellipsis)
+    def expect_ready(self):
+        msg = self.expect(ANSWER_READY, Ellipsis, Ellipsis)
         self.update_times(msg)
 
     def update_times(self, msg):
@@ -74,12 +74,12 @@
         s1, s2 = socket.socketpair()
         ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
         s2.close()
-        msg = self.expect(ANSWER_FORKED, Ellipsis, Ellipsis, Ellipsis)
-        self.update_times(msg)
+        msg = self.expect(ANSWER_FORKED, Ellipsis)
         child_pid = msg.arg3
+        self.expect_ready()
         other = ReplayProcess(child_pid, s1,
                               breakpoints_cache=self.breakpoints_cache)
-        other.update_times(msg)
+        other.expect_ready()
         return other
 
     def close(self):
@@ -89,23 +89,25 @@
         except socket.error:
             pass
 
-    def forward(self, steps):
-        """Move this subprocess forward in time."""
+    def forward(self, steps, breakpoint_mode='b'):
+        """Move this subprocess forward in time.
+        Returns the Breakpoint or None.
+        """
         assert not self.tainted
-        self.send(Message(CMD_FORWARD, steps))
+        self.send(Message(CMD_FORWARD, steps, ord(breakpoint_mode)))
         #
-        msg = self.recv()
-        if msg.cmd == ANSWER_BREAKPOINT:
-            bkpt_num = msg.arg3
+        # record the first ANSWER_BREAKPOINT, drop the others
+        # (in corner cases only could we get more than one)
+        bkpt = None
+        while True:
             msg = self.recv()
-        else:
-            bkpt_num = None
-        assert msg.cmd == ANSWER_STD
+            if msg.cmd != ANSWER_BREAKPOINT:
+                break
+            if bkpt is None:
+                bkpt = Breakpoint(msg.arg1, msg.arg3)
+        assert msg.cmd == ANSWER_READY
         self.update_times(msg)
-        #
-        if bkpt_num is not None:
-            raise Breakpoint(bkpt_num)
-        return msg
+        return bkpt
 
     def print_text_answer(self):
         while True:
@@ -113,7 +115,7 @@
             if msg.cmd == ANSWER_TEXT:
                 sys.stdout.write(msg.extra)
                 sys.stdout.flush()
-            elif msg.cmd == ANSWER_STD:
+            elif msg.cmd == ANSWER_READY:
                 self.update_times(msg)
                 break
             else:
@@ -136,7 +138,7 @@
         child = ReplayProcess(initial_subproc.pid, s1)
         msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
         self.total_stop_points = msg.arg2
-        child.expect_std()
+        child.expect_ready()
 
         self.active = child
         self.paused = {1: child.clone()}     # {time: subprocess}
@@ -148,7 +150,7 @@
     def _check_current_time(self, time):
         assert self.get_current_time() == time
         self.active.send(Message(CMD_FORWARD, 0))
-        return self.active.expect(ANSWER_STD, time, Ellipsis)
+        return self.active.expect(ANSWER_READY, time, Ellipsis)
 
     def get_max_time(self):
         return self.total_stop_points
@@ -165,15 +167,21 @@
     def is_tainted(self):
         return self.active.tainted
 
-    def go_forward(self, steps):
+    def go_forward(self, steps, breakpoint_mode='b'):
         """Go forward, for the given number of 'steps' of time.
 
         If needed, it will leave clones at intermediate times.
         Does not close the active subprocess.  Note that
         is_tainted() must return false in order to use this.
+
+        breakpoint_mode:
+          'b' = regular mode where hitting a breakpoint stops
+          'i' = ignore breakpoints
+          'r' = record the occurrence of a breakpoint but continue
         """
         assert steps >= 0
         self.update_breakpoints()
+        latest_bkpt = None
         while True:
             cur_time = self.get_current_time()
             if cur_time + steps > self.total_stop_points:
@@ -184,11 +192,50 @@
                 break
             assert rel_next_clone >= 0
             if rel_next_clone > 0:
-                self.active.forward(rel_next_clone)
+                bkpt = self.active.forward(rel_next_clone, breakpoint_mode)
+                if breakpoint_mode == 'r':
+                    latest_bkpt = bkpt or latest_bkpt
+                elif bkpt:
+                    raise bkpt
                 steps -= rel_next_clone
             clone = self.active.clone()
             self.paused[clone.current_time] = clone
-        self.active.forward(steps)
+        bkpt = self.active.forward(steps, breakpoint_mode)
+        if breakpoint_mode == 'r':
+            bkpt = bkpt or latest_bkpt
+        if bkpt:
+            raise bkpt
+
+    def go_backward(self, steps):
+        """Go backward, for the given number of 'steps' of time.
+
+        Closes the active process.  Implemented as jump_in_time()
+        and then forward-searching for breakpoint locations (if any).
+        """
+        assert steps >= 0
+        initial_time = self.get_current_time()
+        if not self.breakpoints:
+            self.jump_in_time(initial_time - steps)
+        else:
+            self._backward_search_forward(initial_time - 957, initial_time)
+
+    def _backward_search_forward(self, search_start_time, search_stop_time):
+        while True:
+            self.jump_in_time(search_start_time)
+            search_start_time = self.get_current_time()
+            time_range_to_search = search_stop_time - search_start_time
+            if time_range_to_search <= 0:
+                print "[not found]"
+                return
+            print "[searching %d..%d]\n" % (search_start_time,
+                                            search_stop_time)
+            self.go_forward(time_range_to_search, breakpoint_mode='r')
+            # If at least one breakpoint was found, the Breakpoint
+            # exception is raised with the *last* such breakpoint.
+            # Otherwise, we continue here.  Search farther along a
+            # 3-times-bigger range.
+            search_stop_time = search_start_time
+            search_start_time -= time_range_to_search * 3
 
     def update_breakpoints(self):
         if self.active.breakpoints_cache != self.breakpoints:
@@ -200,7 +247,7 @@
             extra = '\x00'.join(breakpoints)
             self.active.breakpoints_cache = None
             self.active.send(Message(CMD_BREAKPOINTS, extra=extra))
-            self.active.expect_std()
+            self.active.expect_ready()
             self.active.breakpoints_cache = self.breakpoints.copy()
 
     def _resume(self, from_time):
@@ -218,17 +265,8 @@
         if target_time > self.total_stop_points:
             target_time = self.total_stop_points
         self._resume(max(time for time in self.paused if time <= target_time))
-        with self._breakpoints_disabled():
-            self.go_forward(target_time - self.get_current_time())
-
-    @contextmanager
-    def _breakpoints_disabled(self):
-        old =  self.breakpoints
-        self.breakpoints = {}
-        try:
-            yield
-        finally:
-            self.breakpoints = old
+        self.go_forward(target_time - self.get_current_time(),
+                        breakpoint_mode='i')
 
     def close(self):
         """Close all subprocesses.
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -197,7 +197,7 @@
 #define CMD_FORWARD  (-3)
 
 #define ANSWER_INIT       (-20)
-#define ANSWER_STD        (-21)
+#define ANSWER_READY      (-21)
 #define ANSWER_FORKED     (-22)
 #define ANSWER_AT_END     (-23)
 #define ANSWER_BREAKPOINT (-24)
@@ -212,6 +212,9 @@
 static jmp_buf jmp_buf_cancel_execution;
 static void (*pending_after_forward)(void);
 static RPyString *empty_string;
+static uint64_t last_recorded_breakpoint_loc;
+static int last_recorded_breakpoint_num;
+static char breakpoint_mode;
 
 static void attach_gdb(void)
 {
@@ -281,13 +284,6 @@
     write_sock(&c, sizeof(c));
 }
 
-static void answer_std(void)
-{
-    assert(stopped_time != 0);
-    assert(stopped_uid != 0);
-    write_answer(ANSWER_STD, stopped_time, stopped_uid, 0);
-}
-
 static RPyString *make_rpy_string(size_t length)
 {
     RPyString *s = malloc(sizeof(RPyString) + length);
@@ -367,7 +363,6 @@
     empty_string = make_rpy_string(0);
 
     write_answer(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points, 0);
-    pending_after_forward = &answer_std;
 
     /* ignore the SIGCHLD signals so that child processes don't become
        zombies */
@@ -518,11 +513,19 @@
     }
     else {
         /* in the parent */
-        write_answer(ANSWER_FORKED, stopped_time, stopped_uid, child_pid);
+        write_answer(ANSWER_FORKED, child_pid, 0, 0);
         close(child_sockfd);
     }
 }
 
+static void answer_recorded_breakpoint(void)
+{
+    if (last_recorded_breakpoint_loc != 0) {
+        write_answer(ANSWER_BREAKPOINT, last_recorded_breakpoint_loc,
+                     0, last_recorded_breakpoint_num);
+    }
+}
+
 static void command_forward(rpy_revdb_command_t *cmd)
 {
     if (cmd->arg1 < 0) {
@@ -530,7 +533,11 @@
         exit(1);
     }
     rpy_revdb.stop_point_break = stopped_time + cmd->arg1;
-    pending_after_forward = &answer_std;
+    breakpoint_mode = (char)cmd->arg2;
+    if (breakpoint_mode == 'r') {
+        last_recorded_breakpoint_loc = 0;
+        pending_after_forward = &answer_recorded_breakpoint;
+    }
 }
 
 static void command_default(rpy_revdb_command_t *cmd, char *extra)
@@ -551,7 +558,6 @@
         s = make_rpy_string(cmd->extra_size);
         memcpy(_RPyString_AsString(s), extra, cmd->extra_size);
     }
-    pending_after_forward = &answer_std;
     execute_rpy_function(rpy_revdb_command_funcs[i], cmd, s);
 }
 
@@ -562,6 +568,7 @@
         stopped_time = rpy_revdb.stop_point_seen;
         stopped_uid = rpy_revdb.unique_id_seen;
         rpy_revdb.unique_id_seen = (-1ULL) << 63;
+        breakpoint_mode = 0;
 
         if (pending_after_forward) {
             void (*fn)(void) = pending_after_forward;
@@ -570,6 +577,7 @@
         }
         else {
             rpy_revdb_command_t cmd;
+            write_answer(ANSWER_READY, stopped_time, stopped_uid, 0);
             read_sock(&cmd, sizeof(cmd));
 
             char extra[cmd.extra_size + 1];
@@ -655,8 +663,20 @@
                         "debug command\n");
         exit(1);
     }
-    rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
-    write_answer(ANSWER_BREAKPOINT, rpy_revdb.stop_point_break, 0, num);
+
+    switch (breakpoint_mode) {
+    case 'i':
+        return;   /* ignored breakpoints */
+
+    case 'r':     /* record the breakpoint but continue */
+        last_recorded_breakpoint_loc = rpy_revdb.stop_point_seen + 1;
+        last_recorded_breakpoint_num = num;
+        return;
+
+    default:      /* 'b' or '\0' for default handling of breakpoints */
+        rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1;
+        write_answer(ANSWER_BREAKPOINT, rpy_revdb.stop_point_break, 0, num);
+    }
 }
 
 RPY_EXTERN
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -220,7 +220,7 @@
         self.subproc = subproc
         child = ReplayProcess(subproc.pid, s1)
         child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3)
-        child.expect(ANSWER_STD, 1, Ellipsis)
+        child.expect(ANSWER_READY, 1, Ellipsis)
         return child
 
 
@@ -245,7 +245,7 @@
     def test_go(self):
         child = self.replay()
         child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_STD, 3, Ellipsis)
+        child.expect(ANSWER_READY, 3, Ellipsis)
         child.send(Message(CMD_FORWARD, 2))
         child.expect(ANSWER_AT_END)
 
@@ -258,9 +258,9 @@
         child = self.replay()
         child2 = child.clone()
         child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_STD, 3, Ellipsis)
+        child.expect(ANSWER_READY, 3, Ellipsis)
         child2.send(Message(CMD_FORWARD, 1))
-        child2.expect(ANSWER_STD, 2, Ellipsis)
+        child2.expect(ANSWER_READY, 2, Ellipsis)
         #
         child.close()
         child2.close()
@@ -306,8 +306,8 @@
             if extra == 'get-value':
                 revdb.send_answer(100, revdb.current_time(),
                                        revdb.total_time())
-            if extra == 'go-fw':
-                revdb.go_forward(1, went_fw)
+            ## if extra == 'go-fw':
+            ##     revdb.go_forward(1, went_fw)
             ## if cmdline == 'set-break-after-0':
             ##     dbstate.break_after = 0
             ## if cmdline == 'print-id':
@@ -380,11 +380,11 @@
         child.send(Message(1, extra='get-value'))
         child.expect(100, 1, 3)
 
-    def test_go_fw(self):
-        child = self.replay()
-        child.send(Message(1, extra='go-fw'))
-        child.expect(42, 1, -43, -44, 'go-fw')
-        child.expect(120, 2)
-        child.expect(120, 3)
-        child.send(Message(CMD_FORWARD, 0))
-        child.expect(ANSWER_STD, 3, Ellipsis)
+    ## def test_go_fw(self):
+    ##     child = self.replay()
+    ##     child.send(Message(1, extra='go-fw'))
+    ##     child.expect(42, 1, -43, -44, 'go-fw')
+    ##     child.expect(120, 2)
+    ##     child.expect(120, 3)
+    ##     child.send(Message(CMD_FORWARD, 0))
+    ##     child.expect(ANSWER_READY, 3, Ellipsis)
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -16,7 +16,7 @@
             pass
 
         class DBState:
-            break_loop = -1
+            break_loop = -2
         dbstate = DBState()
 
         def blip(cmd, extra):
@@ -32,7 +32,7 @@
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
-                if dbstate.break_loop == i:
+                if i == dbstate.break_loop or i == dbstate.break_loop + 1:
                     revdb.breakpoint(99)
                 revdb.stop_point()
                 print op
@@ -61,11 +61,29 @@
             group.jump_in_time(target_time)
             group._check_current_time(target_time)
 
-    def test_breakpoint(self):
+    def test_breakpoint_b(self):
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
         group.active.send(Message(1, 6, extra='set-breakpoint'))
         group.active.expect(42, 1, -43, -44, 'set-breakpoint')
-        group.active.expect(ANSWER_STD, 1, Ellipsis)
-        e = py.test.raises(Breakpoint, group.go_forward, 10)
+        group.active.expect(ANSWER_READY, 1, Ellipsis)
+        e = py.test.raises(Breakpoint, group.go_forward, 10, 'b')
+        assert e.value.time == 7
         assert e.value.num == 99
         group._check_current_time(7)
+
+    def test_breakpoint_r(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        group.active.send(Message(1, 6, extra='set-breakpoint'))
+        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.expect(ANSWER_READY, 1, Ellipsis)
+        e = py.test.raises(Breakpoint, group.go_forward, 10, 'r')
+        assert e.value.time == 8
+        assert e.value.num == 99
+        group._check_current_time(10)
+
+    def test_breakpoint_i(self):
+        group = ReplayProcessGroup(str(self.exename), self.rdbname)
+        group.active.send(Message(1, 6, extra='set-breakpoint'))
+        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.expect(ANSWER_READY, 1, Ellipsis)
+        group.go_forward(10, 'i')    # does not raise Breakpoint

From pypy.commits at gmail.com  Mon Jun 20 13:06:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 10:06:00 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix
Message-ID: <57682278.d12f1c0a.5ef37.ffffc67a@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85251:34aad1b16720
Date: 2016-06-20 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/34aad1b16720/

Log:	fix

diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -217,18 +217,22 @@
         if not self.breakpoints:
             self.jump_in_time(initial_time - steps)
         else:
-            self._backward_search_forward(initial_time - 957, initial_time)
+            self._backward_search_forward(
+                search_start_time       = initial_time - 957,
+                search_stop_time        = initial_time - 1,
+                search_go_on_until_time = initial_time - steps)
 
-    def _backward_search_forward(self, search_start_time, search_stop_time):
+    def _backward_search_forward(self, search_start_time, search_stop_time,
+                                 search_go_on_until_time=1):
         while True:
-            self.jump_in_time(search_start_time)
+            self.jump_in_time(max(search_start_time, search_go_on_until_time))
             search_start_time = self.get_current_time()
             time_range_to_search = search_stop_time - search_start_time
             if time_range_to_search <= 0:
                 print "[not found]"
                 return
-            print "[searching %d..%d]\n" % (search_start_time,
-                                            search_stop_time)
+            print "[searching %d..%d]" % (search_start_time,
+                                          search_stop_time)
             self.go_forward(time_range_to_search, breakpoint_mode='r')
             # If at least one breakpoint was found, the Breakpoint
             # exception is raised with the *last* such breakpoint.

From pypy.commits at gmail.com  Mon Jun 20 13:23:30 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 20 Jun 2016 10:23:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: set correct types for unpack pyopcodes
Message-ID: <57682692.4346c20a.8225a.0b9f@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85252:9a39ae0f6424
Date: 2016-06-20 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9a39ae0f6424/

Log:	set correct types for unpack pyopcodes

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -254,24 +254,24 @@
                 self.BUILD_LIST(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST_FROM_ARG.index:
                 self.BUILD_LIST_FROM_ARG(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_LIST_UNPACK.index:
+                self.BUILD_LIST_UNPACK(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_MAP.index:
                 self.BUILD_MAP(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_MAP_UNPACK.index:
+                self.BUILD_MAP_UNPACK(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_MAP_UNPACK_WITH_CALL.index:
+                self.BUILD_MAP_UNPACK_WITH_CALL(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_SET.index:
                 self.BUILD_SET(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_SET_UNPACK.index:
                 self.BUILD_SET_UNPACK(oparg, next_instr)
-            elif opcode == opcodedesc.BUILD_TUPLE_UNPACK.index:
-                self.BUILD_TUPLE_UNPACK(oparg, next_instr)
-            elif opcode == opcodedesc.BUILD_LIST_UNPACK.index:
-                self.BUILD_LIST_UNPACK(oparg, next_instr)
-            elif opcode == opcodedesc.BUILD_MAP_UNPACK.index:
-                self.BUILD_MAP_UNPACK(oparg, next_instr)
-            elif opcode == opcodedesc.BUILD_MAP_UNPACK_WITH_CALL.index:
-                self.BUILD_MAP_UNPACK_WITH_CALL(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_SLICE.index:
                 self.BUILD_SLICE(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_TUPLE.index:
                 self.BUILD_TUPLE(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_TUPLE_UNPACK.index:
+                self.BUILD_TUPLE_UNPACK(oparg, next_instr)
             elif opcode == opcodedesc.CALL_FUNCTION.index:
                 self.CALL_FUNCTION(oparg, next_instr)
             elif opcode == opcodedesc.CALL_FUNCTION_KW.index:
@@ -1333,34 +1333,40 @@
     def BUILD_SET_UNPACK(self, itemcount, next_instr):
         w_sum = self.space.newset()
         for i in range(itemcount, 0, -1):
-            self.space.call_method(w_sum, 'update', self.space.peek(i))
-        while itemcount != 0:
-            self.popvalue()
-            itemcount -= 1
+            w_item = self.popvalue()
+            #self.space.peek(i)
+            self.space.call_method(w_sum, 'update', w_item)
+        #while itemcount != 0:
+        #    self.popvalue()
+        #    itemcount -= 1
         self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newset()
+        w_sum = self.space.newtuple()
         for i in range(itemcount, 0, -1):
-            self.space.call_method(w_sum, 'update', self.space.peek(i))
-        while itemcount != 0:
-            self.popvalue()
-            itemcount -= 1
+            w_item = self.popvalue()
+            #self.space.peek(i)
+            self.space.call_method(w_sum, 'update', w_item)
+        #while itemcount != 0:
+        #    self.popvalue()
+        #    itemcount -= 1
         self.pushvalue(w_sum)
         
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newset()
+        w_sum = self.space.newlist()
         for i in range(itemcount, 0, -1):
-            self.space.call_method(w_sum, 'update', self.space.peek(i))
-        while itemcount != 0:
-            self.popvalue()
-            itemcount -= 1
+            w_item = self.popvalue()
+            #self.space.peek(i)
+            self.space.call_method(w_sum, 'update', w_item)
+        #while itemcount != 0:
+        #    self.popvalue()
+        #    itemcount -= 1
         self.pushvalue(w_sum)
         
     #TODO
     #get intersection, store as setentry
     def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr):
-        w_sum = self.space.newset()
+        w_sum = self.space.newdict()
         num_maps = itemcount & 0xff
         function_location = (itemcount >> 8) & 0xff
         for i in range(num_maps, 0, -1):
@@ -1373,12 +1379,14 @@
         self.pushvalue(w_sum)
         
     def BUILD_MAP_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newset()
+        w_sum = self.space.newdict()
         for i in range(itemcount, 0, -1):
-            self.space.call_method(w_sum, 'update', self.space.peek(i))
-        while itemcount != 0:
-            self.popvalue()
-            itemcount -= 1
+            w_item = self.popvalue()
+            #self.space.peek(i)
+            self.space.call_method(w_sum, 'update', w_item)
+        #while itemcount != 0:
+        #    self.popvalue()
+        #    itemcount -= 1
         self.pushvalue(w_sum)
 ### ____________________________________________________________ ###
 

From pypy.commits at gmail.com  Mon Jun 20 14:03:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 11:03:58 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: next/bnext,
 using temporary breakpoints that must trigger if the stack
Message-ID: <5768300e.88c11c0a.7d2dc.ffffa967@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85253:b894a632f9e3
Date: 2016-06-20 20:04 +0200
http://bitbucket.org/pypy/pypy/changeset/b894a632f9e3/

Log:	next/bnext, using temporary breakpoints that must trigger if the
	stack depth is smaller than a limit

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -14,7 +14,9 @@
 CMD_BACKTRACE   = 2
 CMD_LOCALS      = 3
 CMD_BREAKPOINTS = 4
+CMD_MOREINFO    = 5
 ANSWER_TEXT     = 20
+ANSWER_MOREINFO = 21
 
 
 def stop_point():
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -91,26 +91,34 @@
     def move_forward(self, steps):
         try:
             self.pgroup.go_forward(steps)
+            return True
         except Breakpoint as b:
             self.hit_breakpoint(b)
+            return False
 
     def move_backward(self, steps):
         try:
             self.pgroup.go_backward(steps)
+            return True
         except Breakpoint as b:
             self.hit_breakpoint(b)
+            return False
 
     def hit_breakpoint(self, b):
-        print 'Hit breakpoint %d' % (b.num,)
+        if b.num != -1:
+            print 'Hit breakpoint %d' % (b.num,)
         if self.pgroup.get_current_time() != b.time:
             self.pgroup.jump_in_time(b.time)
 
+    def remove_tainting(self):
+        if self.pgroup.is_tainted():
+            self.pgroup.jump_in_time(self.pgroup.get_current_time())
+            assert not self.pgroup.is_tainted()
+
     def command_step(self, argument):
         """Run forward ARG steps (default 1)"""
         arg = int(argument or '1')
-        if self.pgroup.is_tainted():
-            self.pgroup.jump_in_time(self.pgroup.get_current_time())
-            assert not self.pgroup.is_tainted()
+        self.remove_tainting()
         self.move_forward(arg)
     command_s = command_step
 
@@ -120,6 +128,43 @@
         self.move_backward(arg)
     command_bs = command_bstep
 
+    def command_next(self, argument):
+        """Run forward for one step, skipping calls"""
+        self.remove_tainting()
+        depth1 = self.pgroup.get_stack_depth()
+        if self.move_forward(1):
+            depth2 = self.pgroup.get_stack_depth()
+            if depth2 > depth1:
+                # If, after running one step, the stack depth is greater
+                # than before, then continue until it is back to what it was.
+                # Can't do it more directly because the "breakpoint" of
+                # stack_depth is only checked for on function enters and
+                # returns (which simplifies and speeds up things for the
+                # RPython code).
+                b = self.pgroup.edit_breakpoints()
+                b.stack_depth = depth1 + 1   # must be < depth1+1
+                try:
+                    self.command_continue('')
+                finally:
+                    b.stack_depth = 0
+    command_n = command_next
+
+    def command_bnext(self, argument):
+        """Run backward for one step, skipping calls"""
+        depth1 = self.pgroup.get_stack_depth()
+        if self.move_backward(1):
+            depth2 = self.pgroup.get_stack_depth()
+            if depth2 > depth1:
+                # If, after running one bstep, the stack depth is greater
+                # than before, then bcontinue until it is back to what it was.
+                b = self.pgroup.edit_breakpoints()
+                b.stack_depth = depth1 + 1   # must be < depth1+1
+                try:
+                    self.command_bcontinue('')
+                finally:
+                    b.stack_depth = 0
+    command_bn = command_bnext
+
     def command_continue(self, argument):
         """Run forward"""
         self.move_forward(self.pgroup.get_max_time() -
@@ -147,18 +192,20 @@
 
     def command_break(self, argument):
         """Add a breakpoint"""
+        b = self.pgroup.edit_breakpoints()
         new = 1
-        while new in self.pgroup.breakpoints:
+        while new in b.num2name:
             new += 1
-        self.pgroup.breakpoints[new] = argument
+        b.num2name[new] = argument
         print "Breakpoint %d added" % (new,)
     command_b = command_break
 
     def command_delete(self, argument):
         """Delete a breakpoint"""
         arg = int(argument)
-        if arg not in self.pgroup.breakpoints:
-            print "No breakpoint number %d" % (new,)
+        b = self.pgroup.edit_breakpoints()
+        if arg not in b.num2name:
+            print "No breakpoint number %d" % (arg,)
         else:
-            del self.pgroup.breakpoints[arg]
-            print "Breakpoint %d deleted" % (new,)
+            del b.num2name[arg]
+            print "Breakpoint %d deleted" % (arg,)
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -12,8 +12,9 @@
 CMD_PRINT       = 1    # Message(CMD_PRINT, extra=expression)
 CMD_BACKTRACE   = 2    # Message(CMD_BACKTRACE)
 CMD_LOCALS      = 3    # Message(CMD_LOCALS)
-CMD_BREAKPOINTS = 4    # Message(CMD_BREAKPOINTS, extra="\0-separated names")
-
+CMD_BREAKPOINTS = 4    # Message(CMD_BREAKPOINTS, stack_depth,
+                       #         extra="\0-separated names")
+CMD_MOREINFO    = 5    # Message(CMD_MOREINFO)
 
 
 # the first message sent by the first child:
@@ -45,6 +46,10 @@
 #    Message(ANSWER_TEXT, extra=text)
 ANSWER_TEXT       = 20
 
+# sent after CMD_MOREINFO:
+#    Message(ANSWER_MOREINFO, stack_depth)
+ANSWER_MOREINFO   = 21
+
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -11,6 +11,35 @@
         self.time = time
         self.num = num
 
+    def __repr__(self):
+        return 'Breakpoint(%d, %d)' % (self.time, self.num)
+
+
+class AllBreakpoints(object):
+
+    def __init__(self):
+        self.num2name = {}    # {small number: function name}
+        self.stack_depth = 0     # breaks if the depth becomes lower than this
+
+    def __repr__(self):
+        return 'AllBreakpoints(%r, %d)' % (self.num2name, self.stack_depth)
+
+    def __eq__(self, other):
+        return (self.num2name == other.num2name and
+                self.stack_depth == other.stack_depth)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def is_empty(self):
+        return len(self.num2name) == 0 and self.stack_depth == 0
+
+    def duplicate(self):
+        a = AllBreakpoints()
+        a.num2name.update(self.num2name)
+        a.stack_depth = self.stack_depth
+        return a
+
 
 class ReplayProcess(object):
     """Represent one replaying subprocess.
@@ -18,11 +47,11 @@
     It can be either the one started with --revdb-replay, or a fork.
     """
 
-    def __init__(self, pid, control_socket, breakpoints_cache={}):
+    def __init__(self, pid, control_socket, breakpoints_cache=AllBreakpoints()):
         self.pid = pid
         self.control_socket = control_socket
         self.tainted = False
-        self.breakpoints_cache = breakpoints_cache    # don't change this dict
+        self.breakpoints_cache = breakpoints_cache    # don't mutate this
 
     def _recv_all(self, size):
         pieces = []
@@ -75,7 +104,7 @@
         ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
         s2.close()
         msg = self.expect(ANSWER_FORKED, Ellipsis)
-        child_pid = msg.arg3
+        child_pid = msg.arg1
         self.expect_ready()
         other = ReplayProcess(child_pid, s1,
                               breakpoints_cache=self.breakpoints_cache)
@@ -142,7 +171,7 @@
 
         self.active = child
         self.paused = {1: child.clone()}     # {time: subprocess}
-        self.breakpoints = {}
+        self.all_breakpoints = AllBreakpoints()
 
     def get_current_time(self):
         return self.active.current_time
@@ -214,7 +243,7 @@
         """
         assert steps >= 0
         initial_time = self.get_current_time()
-        if not self.breakpoints:
+        if self.all_breakpoints.is_empty():
             self.jump_in_time(initial_time - steps)
         else:
             self._backward_search_forward(
@@ -242,17 +271,19 @@
             search_start_time -= time_range_to_search * 3
 
     def update_breakpoints(self):
-        if self.active.breakpoints_cache != self.breakpoints:
-            if self.breakpoints:
-                breakpoints = [self.breakpoints.get(n, '')
-                               for n in range(max(self.breakpoints) + 1)]
-            else:
-                breakpoints = []
-            extra = '\x00'.join(breakpoints)
-            self.active.breakpoints_cache = None
-            self.active.send(Message(CMD_BREAKPOINTS, extra=extra))
-            self.active.expect_ready()
-            self.active.breakpoints_cache = self.breakpoints.copy()
+        if self.active.breakpoints_cache == self.all_breakpoints:
+            return
+        num2name = self.all_breakpoints.num2name
+        flat = []
+        if num2name:
+            flat = [num2name.get(n, '') for n in range(max(num2name) + 1)]
+        arg1 = self.all_breakpoints.stack_depth
+        extra = '\x00'.join(flat)
+        #
+        self.active.breakpoints_cache = None
+        self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra))
+        self.active.expect_ready()
+        self.active.breakpoints_cache = self.all_breakpoints.duplicate()
 
     def _resume(self, from_time):
         clone_me = self.paused[from_time]
@@ -296,3 +327,12 @@
         """
         self.active.send(Message(CMD_LOCALS))
         self.active.print_text_answer()
+
+    def edit_breakpoints(self):
+        return self.all_breakpoints
+
+    def get_stack_depth(self):
+        self.active.send(Message(CMD_MOREINFO))
+        msg = self.active.expect(ANSWER_MOREINFO, Ellipsis)
+        self.active.expect_ready()
+        return msg.arg1

From pypy.commits at gmail.com  Mon Jun 20 14:10:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 11:10:46 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: finish/bfinish (tweaks pending)
Message-ID: <576831a6.9a4a1c0a.2d61d.ffffe2ee@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85254:b7a90b2cd17c
Date: 2016-06-20 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/b7a90b2cd17c/

Log:	finish/bfinish (tweaks pending)

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -1,6 +1,8 @@
 import sys, re
 import subprocess, socket
 import traceback
+from contextlib import contextmanager
+
 from rpython.translator.revdb.process import ReplayProcessGroup, maxint64
 from rpython.translator.revdb.process import Breakpoint
 
@@ -128,6 +130,16 @@
         self.move_backward(arg)
     command_bs = command_bstep
 
+    @contextmanager
+    def _stack_depth_break(self, range_stop):
+        # add temporarily a breakpoint for "stack_depth < range_stop"
+        b = self.pgroup.edit_breakpoints()
+        b.stack_depth = range_stop
+        try:
+            yield
+        finally:
+            b.stack_depth = 0
+
     def command_next(self, argument):
         """Run forward for one step, skipping calls"""
         self.remove_tainting()
@@ -141,12 +153,8 @@
                 # stack_depth is only checked for on function enters and
                 # returns (which simplifies and speeds up things for the
                 # RPython code).
-                b = self.pgroup.edit_breakpoints()
-                b.stack_depth = depth1 + 1   # must be < depth1+1
-                try:
+                with self._stack_depth_break(depth1 + 1):
                     self.command_continue('')
-                finally:
-                    b.stack_depth = 0
     command_n = command_next
 
     def command_bnext(self, argument):
@@ -157,14 +165,23 @@
             if depth2 > depth1:
                 # If, after running one bstep, the stack depth is greater
                 # than before, then bcontinue until it is back to what it was.
-                b = self.pgroup.edit_breakpoints()
-                b.stack_depth = depth1 + 1   # must be < depth1+1
-                try:
+                with self._stack_depth_break(depth1 + 1):
+                    # XXX check: 'bnext' stops at the first opcode *inside* the
+                    # function, should stop just before
                     self.command_bcontinue('')
-                finally:
-                    b.stack_depth = 0
     command_bn = command_bnext
 
+    def command_finish(self, argument):
+        self.remove_tainting()
+        with self._stack_depth_break(self.pgroup.get_stack_depth()):
+            self.command_continue('')
+
+    def command_bfinish(self, argument):
+        # XXX check: 'bfinish' stops at the first opcode *inside* the
+        # function, should stop just before
+        with self._stack_depth_break(self.pgroup.get_stack_depth()):
+            self.command_bcontinue('')
+
     def command_continue(self, argument):
         """Run forward"""
         self.move_forward(self.pgroup.get_max_time() -

From pypy.commits at gmail.com  Mon Jun 20 14:17:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Jun 2016 11:17:21 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Tweaks
Message-ID: <57683331.4a9bc20a.b68d3.fffff625@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85255:9264d040e7a2
Date: 2016-06-20 20:18 +0200
http://bitbucket.org/pypy/pypy/changeset/9264d040e7a2/

Log:	Tweaks

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -103,12 +103,14 @@
             self.pgroup.go_backward(steps)
             return True
         except Breakpoint as b:
-            self.hit_breakpoint(b)
+            self.hit_breakpoint(b, backward=True)
             return False
 
-    def hit_breakpoint(self, b):
+    def hit_breakpoint(self, b, backward=False):
         if b.num != -1:
             print 'Hit breakpoint %d' % (b.num,)
+        elif backward:
+            b.time -= 1
         if self.pgroup.get_current_time() != b.time:
             self.pgroup.jump_in_time(b.time)
 
@@ -166,8 +168,6 @@
                 # If, after running one bstep, the stack depth is greater
                 # than before, then bcontinue until it is back to what it was.
                 with self._stack_depth_break(depth1 + 1):
-                    # XXX check: 'bnext' stops at the first opcode *inside* the
-                    # function, should stop just before
                     self.command_bcontinue('')
     command_bn = command_bnext
 
@@ -177,8 +177,6 @@
             self.command_continue('')
 
     def command_bfinish(self, argument):
-        # XXX check: 'bfinish' stops at the first opcode *inside* the
-        # function, should stop just before
         with self._stack_depth_break(self.pgroup.get_stack_depth()):
             self.command_bcontinue('')
 
@@ -209,6 +207,9 @@
 
     def command_break(self, argument):
         """Add a breakpoint"""
+        if not argument:
+            print "Break where?"
+            return
         b = self.pgroup.edit_breakpoints()
         new = 1
         while new in b.num2name:

From pypy.commits at gmail.com  Mon Jun 20 15:09:51 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Jun 2016 12:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: merge default into branch
Message-ID: <57683f7f.e457c20a.47e40.ffffcd14@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85256:b6a30f9a9f1b
Date: 2016-06-17 09:16 +0300
http://bitbucket.org/pypy/pypy/changeset/b6a30f9a9f1b/

Log:	merge default into branch

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -35,3 +35,5 @@
 Simplify handling of interp-level tests and make it more forward-
 compatible.
 
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -43,15 +43,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -195,7 +186,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -463,9 +454,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -508,12 +499,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,9 +1,9 @@
 # Edit these appropriately before running this script
 maj=5
 min=3
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1

From pypy.commits at gmail.com  Mon Jun 20 15:09:53 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Jun 2016 12:09:53 -0700 (PDT)
Subject: [pypy-commit] pypy rw-PyString_AS_STRING: close branch to be merged
Message-ID: <57683f81.49c51c0a.69fe2.ffffabb4@mx.google.com>

Author: Matti Picus 
Branch: rw-PyString_AS_STRING
Changeset: r85257:e867f5b14d22
Date: 2016-06-20 21:51 +0300
http://bitbucket.org/pypy/pypy/changeset/e867f5b14d22/

Log:	close branch to be merged


From pypy.commits at gmail.com  Mon Jun 20 15:09:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Jun 2016 12:09:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge rw-PyString_AS_STRING which
 allows writing to a non-forced char *
Message-ID: <57683f84.4a79c20a.a7e6b.06ca@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85258:38f3eaef5d7b
Date: 2016-06-20 21:52 +0300
http://bitbucket.org/pypy/pypy/changeset/38f3eaef5d7b/

Log:	merge rw-PyString_AS_STRING which allows writing to a non-forced
	char *

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1202,8 +1202,6 @@
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
-            if space.is_w(w_type, space.w_str):
-                pto.c_tp_itemsize = 1
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -6,7 +6,9 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref)
+    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
+    pyobj_has_w_obj)
+from pypy.objspace.std.bytesobject import W_BytesObject
 
 ##
 ## Implementation of PyStringObject
@@ -16,7 +18,7 @@
 ## -----------
 ##
 ## PyString_AsString() must return a (non-movable) pointer to the underlying
-## buffer, whereas pypy strings are movable.  C code may temporarily store
+## ob_sval, whereas pypy strings are movable.  C code may temporarily store
 ## this address and use it, as long as it owns a reference to the PyObject.
 ## There is no "release" function to specify that the pointer is not needed
 ## any more.
@@ -28,7 +30,7 @@
 ## --------
 ##
 ## PyStringObject contains two additional members: the ob_size and a pointer to a
-## char buffer; it may be NULL.
+## char ob_sval; it may be NULL.
 ##
 ## - A string allocated by pypy will be converted into a PyStringObject with a
 ##   NULL buffer.  The first time PyString_AsString() is called, memory is
@@ -41,6 +43,9 @@
 ##   the pypy string is created, and added to the global map of tracked
 ##   objects.  The buffer is then supposed to be immutable.
 ##
+##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
+##   there is no corresponding pypy object for the string
+##
 ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
 ##   similar object.
 ##
@@ -53,7 +58,7 @@
 PyStringObjectStruct = lltype.ForwardReference()
 PyStringObject = lltype.Ptr(PyStringObjectStruct)
 PyStringObjectFields = PyVarObjectFields + \
-    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP))
+    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char)))
 cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct)
 
 @bootstrap_function
@@ -69,44 +74,43 @@
 
 def new_empty_str(space, length):
     """
-    Allocate a PyStringObject and its buffer, but without a corresponding
-    interpreter object.  The buffer may be mutated, until string_realize() is
+    Allocate a PyStringObject and its ob_sval, but without a corresponding
+    interpreter object.  The ob_sval may be mutated, until string_realize() is
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_str.layout.typedef)
-    py_obj = typedescr.allocate(space, space.w_str)
+    py_obj = typedescr.allocate(space, space.w_str, length)
     py_str = rffi.cast(PyStringObject, py_obj)
-
-    buflen = length + 1
-    py_str.c_ob_size = length
-    py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+    py_str.c_ob_shash = -1
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
 def string_attach(space, py_obj, w_obj):
     """
-    Fills a newly allocated PyStringObject with the given string object. The
-    buffer must not be modified.
+    Copy RPython string object contents to a PyStringObject. The
+    c_ob_sval must not be modified.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    py_str.c_ob_size = len(space.str_w(w_obj))
-    py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
+    s = space.str_w(w_obj)
+    if py_str.c_ob_size  < len(s):
+        raise oefmt(space.w_ValueError,
+            "string_attach called on object with ob_size %d but trying to store %d",
+            py_str.c_ob_size, len(s)) 
+    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def string_realize(space, py_obj):
     """
-    Creates the string in the interpreter. The PyStringObject buffer must not
+    Creates the string in the interpreter. The PyStringObject ob_sval must not
     be modified after this call.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    if not py_str.c_buffer:
-        py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                    flavor='raw', zero=True)
-    s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
-    w_obj = space.wrap(s)
+    s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(W_BytesObject, w_type)
+    w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
@@ -116,9 +120,6 @@
 def string_dealloc(space, py_obj):
     """Frees allocated PyStringObject resources.
     """
-    py_str = rffi.cast(PyStringObject, py_obj)
-    if py_str.c_buffer:
-        lltype.free(py_str.c_buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -139,6 +140,9 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
+    return _PyString_AsString(space, ref)
+
+def _PyString_AsString(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         pass    # typecheck returned "ok" without forcing 'ref' at all
     elif not PyString_Check(space, ref):   # otherwise, use the alternate way
@@ -151,15 +155,23 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    return ref_str.c_buffer
+    if not pyobj_has_w_obj(ref):
+        # XXX Force the ref?
+        string_realize(space, ref)
+    return ref_str.c_ob_sval
+
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
+def PyString_AS_STRING(space, void_ref):
+    ref = rffi.cast(PyObject, void_ref)
+    # if no w_str is associated with this ref,
+    # return the c-level ptr as RW
+    if not pyobj_has_w_obj(ref):
+        py_str = rffi.cast(PyStringObject, ref)
+        return py_str.c_ob_sval
+    return _PyString_AsString(space, ref)
 
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
-def PyString_AsStringAndSize(space, ref, buffer, length):
+def PyString_AsStringAndSize(space, ref, data, length):
     if not PyString_Check(space, ref):
         from pypy.module.cpyext.unicodeobject import (
             PyUnicode_Check, _PyUnicode_AsDefaultEncodedString)
@@ -169,18 +181,16 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
+    if not pyobj_has_w_obj(ref):
+        # force the ref
+        string_realize(space, ref)
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    buffer[0] = ref_str.c_buffer
+    data[0] = ref_str.c_ob_sval
     if length:
         length[0] = ref_str.c_ob_size
     else:
         i = 0
-        while ref_str.c_buffer[i] != '\0':
+        while ref_str.c_ob_sval[i] != '\0':
             i += 1
         if i != ref_str.c_ob_size:
             raise oefmt(space.w_TypeError,
@@ -209,10 +219,10 @@
     set to NULL, a memory exception is set, and -1 is returned.
     """
     # XXX always create a new string so far
-    py_str = rffi.cast(PyStringObject, ref[0])
-    if not py_str.c_buffer:
+    if pyobj_has_w_obj(ref[0]):
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
+    py_str = rffi.cast(PyStringObject, ref[0])
     try:
         py_newstr = new_empty_str(space, newsize)
     except MemoryError:
@@ -224,7 +234,7 @@
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newstr.c_buffer[i] = py_str.c_buffer[i]
+        py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newstr)
     return 0
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -10,7 +10,6 @@
 #include 
 
 #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op))
-#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op))
 /*
 Type PyStringObject represents a character string.  An extra zero byte is
 reserved at the end to ensure it is zero-terminated, but a size is
@@ -41,12 +40,11 @@
     PyObject_VAR_HEAD
     long ob_shash;
     int ob_sstate;
-    char * buffer; /* change the name from cpython so all non-api c access is thwarted */
+    char ob_sval[1]; 
 
     /* Invariants 
-     * (not relevant in PyPy, all stringobjects are backed by a pypy object)
-     *     buffer contains space for 'ob_size+1' elements.
-     *     buffer[ob_size] == 0.
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
      *     ob_shash is the hash of the string or -1 if not computed yet.
      *     ob_sstate != 0 iff the string object is in stringobject.c's
      *       'interned' dictionary; in this case the two references
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    INTERPLEVEL_API)
+    INTERPLEVEL_API, PyVarObject)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.objectobject import W_ObjectObject
@@ -47,13 +47,16 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount and w_type is not space.w_str:
+        if pytype.c_tp_itemsize:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
+        if pytype.c_tp_itemsize:
+            pyvarobj = rffi.cast(PyVarObject, pyobj)
+            pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
         #pyobj.c_ob_pypy_link should get assigned very quickly
         pyobj.c_ob_type = pytype
@@ -152,13 +155,18 @@
 class InvalidPointerException(Exception):
     pass
 
-def create_ref(space, w_obj, itemcount=0):
+def create_ref(space, w_obj):
     """
     Allocates a PyObject, and fills its fields with info from the given
     interpreter object.
     """
     w_type = space.type(w_obj)
+    pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
+    if pytype.c_tp_itemsize != 0:
+        itemcount = space.len_w(w_obj) # PyStringObject and subclasses
+    else:
+        itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)
     track_reference(space, py_obj, w_obj)
     #
diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c
--- a/pypy/module/cpyext/src/stringobject.c
+++ b/pypy/module/cpyext/src/stringobject.c
@@ -107,7 +107,7 @@
     if (!string)
         return NULL;
 
-    s = PyString_AsString(string);
+    s = PyString_AS_STRING(string);
 
     for (f = format; *f; f++) {
         if (*f == '%') {
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -25,14 +25,13 @@
             ("test_Size", "METH_NOARGS",
              """
                  PyObject* s = PyString_FromString("Hello world");
-                 int result = 0;
+                 int result;
                  size_t expected_size;
 
-                 if(PyString_Size(s) == 11) {
-                     result = 1;
-                 }
+                 result = PyString_Size(s);
+                
                  #ifdef PYPY_VERSION
-                    expected_size = sizeof(void*)*7;
+                    expected_size = 48;
                  #elif defined Py_DEBUG
                     expected_size = 53;
                  #else
@@ -44,7 +43,7 @@
                      result = 0;
                  }
                  Py_DECREF(s);
-                 return PyBool_FromLong(result);
+                 return PyLong_FromLong(result);
              """),
             ("test_Size_exception", "METH_NOARGS",
              """
@@ -60,7 +59,7 @@
              """)], prologue='#include ')
         assert module.get_hello1() == 'Hello world'
         assert module.get_hello2() == 'Hello world'
-        assert module.test_Size()
+        assert module.test_Size() == 11
         raises(TypeError, module.test_Size_exception)
 
         assert module.test_is_string("")
@@ -81,7 +80,7 @@
                  if (t == NULL)
                     return NULL;
                  Py_DECREF(t);
-                 c = PyString_AsString(s);
+                 c = PyString_AS_STRING(s);
                  c[0] = 'a';
                  c[1] = 'b';
                  c[2] = 0;
@@ -110,14 +109,23 @@
                 obj = (PyStringObject*)type->tp_alloc(type, 10);
                 if (PyString_GET_SIZE(obj) != 10)
                     return PyLong_FromLong(PyString_GET_SIZE(obj));
-                /* cannot work, there is only RO access
-                memcpy(PyString_AS_STRING(obj), "works", 6); */
+                /* cannot work, there is only RO access */
+                memcpy(PyString_AS_STRING(obj), "works", 6);
                 Py_INCREF(obj);
                 return (PyObject*)obj;
              """),
+            ('alloc_rw', "METH_NOARGS",
+             '''
+                PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
+                char * buf = PyString_AS_STRING(obj);
+                memcpy(PyString_AS_STRING(obj), "works", 6);
+                return (PyObject*)obj;
+             '''),
             ])
+        s = module.alloc_rw()
+        assert s == 'works' + '\x00' * 5
         s = module.tpalloc()
-        assert s == '\x00' * 10
+        assert s == 'works' + '\x00' * 5
 
     def test_AsString(self):
         module = self.import_extension('foo', [
@@ -332,27 +340,127 @@
         # doesn't really test, but if printf is enabled will prove sstate
         assert module.test_sstate()
 
+    def test_subclass(self):
+        # taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        module = self.import_extension('bar', [
+            ("newsubstr", "METH_O",
+             """
+                PyObject * obj;
+                char * data;
+                int len;
+                PyType_Ready(&PyStringArrType_Type);
+                
+                data = PyString_AS_STRING(args);
+                len = PyString_GET_SIZE(args);
+                if (data == NULL || len < 1)
+                    Py_RETURN_NONE;
+                obj = PyArray_Scalar(data, len);
+                return obj;
+             """),
+            ], prologue="""
+                #include 
+                PyTypeObject PyStringArrType_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "bar.string_",                /* tp_name*/
+                    sizeof(PyStringObject), /* tp_basicsize*/
+                    0                             /* tp_itemsize */
+                    };
+
+                    static PyObject *
+                    stringtype_repr(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    static PyObject *
+                    stringtype_str(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    PyObject *
+                    PyArray_Scalar(char *data, int n)
+                    {
+                        PyTypeObject *type = &PyStringArrType_Type;
+                        PyObject *obj;
+                        void *destptr;
+                        int type_num;
+                        int itemsize = n;
+                        obj = type->tp_alloc(type, itemsize);
+                        if (obj == NULL) {
+                            return NULL;
+                        }
+                        destptr = PyString_AS_STRING(obj);
+                        ((PyStringObject *)obj)->ob_shash = -1;
+                        memcpy(destptr, data, itemsize);
+                        return obj;
+                    }
+            """, more_init = '''
+                PyStringArrType_Type.tp_alloc = NULL;
+                PyStringArrType_Type.tp_free = NULL;
+
+                PyStringArrType_Type.tp_repr = stringtype_repr;
+                PyStringArrType_Type.tp_str = stringtype_str;
+                PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
+                PyStringArrType_Type.tp_itemsize = sizeof(char);
+                PyStringArrType_Type.tp_base = &PyString_Type;
+            ''')
+
+        a = module.newsubstr('abc')
+        assert type(a).__name__ == 'string_'
+        assert a == 'abc'
 
 class TestString(BaseApiTest):
     def test_string_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_buffer[0] = 'a'
-        py_str.c_buffer[1] = 'b'
-        py_str.c_buffer[2] = 'c'
+        py_str.c_ob_sval[0] = 'a'
+        py_str.c_ob_sval[1] = 'b'
+        py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 3)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[3] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 10)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[10] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -657,6 +657,8 @@
         pto.c_tp_dealloc = llhelper(
             subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
+    if space.is_w(w_type, space.w_str):
+        pto.c_tp_itemsize = 1
     # buffer protocol
     setup_buffer_procs(space, w_type, pto)
 
@@ -695,6 +697,8 @@
     if pto.c_tp_base:
         if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize:
             pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize
+        if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
+            pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
     # will be filled later on with the correct value
     # may not be 0
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -77,7 +77,9 @@
     """
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
     s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length)
-    w_obj = space.wrap(s)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
+    w_obj.__init__(s)
     py_uni.c_hash = space.hash_w(w_obj)
     track_reference(space, py_obj, w_obj)
     return w_obj

From pypy.commits at gmail.com  Mon Jun 20 15:09:57 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Jun 2016 12:09:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <57683f85.4d9a1c0a.dd3dc.5f42@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r85259:31f278535f89
Date: 2016-06-20 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/31f278535f89/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,6 +1,6 @@
-=========================
+==========================
 What's new in PyPy2.7 5.3+
-=========================
+==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
@@ -37,3 +37,8 @@
 
 .. branch: pyfile-tell
 Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type

From pypy.commits at gmail.com  Mon Jun 20 15:13:32 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 12:13:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: backout a02677ec50cc: does not work,
 actually
Message-ID: <5768405c.09f6c20a.38379.553f@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85260:ca5b44ad4e30
Date: 2016-06-20 20:12 +0100
http://bitbucket.org/pypy/pypy/changeset/ca5b44ad4e30/

Log:	backout a02677ec50cc: does not work, actually

diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test/test_structseq.py
--- a/pypy/module/cpyext/test/test_structseq.py
+++ b/pypy/module/cpyext/test/test_structseq.py
@@ -6,6 +6,7 @@
 
 class AppTestStructSeq(AppTestCpythonExtensionBase):
     def test_StructSeq(self):
+        skip("XXX: https://bugs.pypy.org/issue1557")
         module = self.import_extension('foo',
         prologue="""
             #include 

From pypy.commits at gmail.com  Mon Jun 20 15:19:22 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 12:19:22 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: hg merge testing-default
Message-ID: <576841ba.c5461c0a.a3aea.ffff8502@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85261:9e634693f1eb
Date: 2016-06-20 19:10 +0100
http://bitbucket.org/pypy/pypy/changeset/9e634693f1eb/

Log:	hg merge testing-default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -19,3 +19,6 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,21 +2,10 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import int_c_div, int_c_mod
 from rpython.rlib import jit
 
 
-# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
-#     because now it expects only Python-style divisions, not the
-#     C-style divisions of these two ll operations
- at jit.dont_look_inside
-def _int_floordiv(n, m):
-    return llop.int_floordiv(lltype.Signed, n, m)
-
- at jit.dont_look_inside
-def _int_mod(n, m):
-    return llop.int_mod(lltype.Signed, n, m)
-
-
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
     return space.wrap(llop.int_add(lltype.Signed, n, m))
@@ -31,11 +20,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(_int_floordiv(n, m))
+    return space.wrap(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(_int_mod(n, m))
+    return space.wrap(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -220,6 +220,11 @@
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise oefmt(space.w_SystemError,
                         "libffi failed to build this callback")
+        if closure_ptr.c_user_data != unique_id:
+            raise oefmt(space.w_SystemError,
+                "ffi_prep_closure(): bad user_data (it seems that the "
+                "version of the libffi library seen at runtime is "
+                "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
         jitdriver1.jit_merge_point(callback=self,
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -802,6 +802,21 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
+def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+    from rpython.rlib import rgil
+    # see "Handling of the GIL" above
+    assert cpyext_glob_tid_ptr[0] == 0
+    if pygilstate_release:
+        from pypy.module.cpyext import pystate
+        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
+    else:
+        unlock = gil_release or _gil_auto
+    if unlock:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
+
+
 def make_wrapper_second_level(space, argtypesw, restype,
                               result_kind, error_value, gil):
     from rpython.rlib import rgil
@@ -829,6 +844,7 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
+        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -841,7 +857,6 @@
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
-            from pypy.module.cpyext import pystate
             if cpyext_glob_tid_ptr[0] == tid:
                 cpyext_glob_tid_ptr[0] = 0
                 args += (pystate.PyGILState_LOCKED,)
@@ -852,6 +867,10 @@
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
+        if pygilstate_release:
+            gilstate = rffi.cast(lltype.Signed, args[-1])
+        else:
+            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -921,24 +940,13 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
+            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        # see "Handling of the GIL" above
-        assert cpyext_glob_tid_ptr[0] == 0
-        if pygilstate_release:
-            from pypy.module.cpyext import pystate
-            arg = rffi.cast(lltype.Signed, args[-1])
-            unlock = (arg == pystate.PyGILState_UNLOCKED)
-        else:
-            unlock = gil_release or _gil_auto
-        if unlock:
-            rgil.release()
-        else:
-            cpyext_glob_tid_ptr[0] = tid
-
+        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -211,6 +211,7 @@
 PyGILState_STATE = rffi.INT
 PyGILState_LOCKED = 0
 PyGILState_UNLOCKED = 1
+PyGILState_IGNORE = 2
 
 ExecutionContext.cpyext_gilstate_counter_noleave = 0
 
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import os
 import pytest
 
 def pytest_configure(config):
@@ -23,3 +23,14 @@
 def pytest_funcarg__api(request):
     return request.cls.api
 
+if os.name == 'nt':
+    @pytest.yield_fixture(autouse=True, scope='session')
+    def prevent_dialog_box():
+        """Do not open dreaded dialog box on segfault on Windows"""
+        import ctypes
+        SEM_NOGPFAULTERRORBOX = 0x0002  # From MSDN
+        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
+        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
+        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
+        yield
+        ctypes.windll.kernel32.SetErrorMode(old_err_mode)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -48,15 +48,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -200,7 +191,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -468,9 +459,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -513,12 +504,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/
diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/support.py
@@ -0,0 +1,68 @@
+import os
+import py
+from sys import platform
+
+if os.name != 'nt':
+    so_ext = 'so'
+else:
+    so_ext = 'dll'
+
+def c_compile(cfilenames, outputfilename,
+        compile_extra=None, link_extra=None,
+        include_dirs=None, libraries=None, library_dirs=None):
+    compile_extra = compile_extra or []
+    link_extra = link_extra or []
+    include_dirs = include_dirs or []
+    libraries = libraries or []
+    library_dirs = library_dirs or []
+    if platform == 'win32':
+        link_extra = link_extra + ['/DEBUG'] # generate .pdb file
+    if platform == 'darwin':
+        # support Fink & Darwinports
+        for s in ('/sw/', '/opt/local/'):
+            if (s + 'include' not in include_dirs
+                    and os.path.exists(s + 'include')):
+                include_dirs.append(s + 'include')
+            if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
+                library_dirs.append(s + 'lib')
+
+    outputfilename = py.path.local(outputfilename).new(ext=so_ext)
+    saved_environ = os.environ.copy()
+    try:
+        _build(
+            cfilenames, outputfilename,
+            compile_extra, link_extra,
+            include_dirs, libraries, library_dirs)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
+    return outputfilename
+
+def _build(cfilenames, outputfilename, compile_extra, link_extra,
+        include_dirs, libraries, library_dirs):
+    from distutils.ccompiler import new_compiler
+    from distutils import sysconfig
+    compiler = new_compiler(force=1)
+    sysconfig.customize_compiler(compiler) # XXX
+    objects = []
+    for cfile in cfilenames:
+        cfile = py.path.local(cfile)
+        old = cfile.dirpath().chdir()
+        try:
+            res = compiler.compile([cfile.basename],
+                include_dirs=include_dirs, extra_preargs=compile_extra)
+            assert len(res) == 1
+            cobjfile = py.path.local(res[0])
+            assert cobjfile.check()
+            objects.append(str(cobjfile))
+        finally:
+            old.chdir()
+
+    compiler.link_shared_object(
+        objects, str(outputfilename),
+        libraries=libraries,
+        extra_preargs=link_extra,
+        library_dirs=library_dirs)
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -31,7 +31,7 @@
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
                      printf("tp_basicsize==%ld\\n",
-                            (long)s->ob_type->tp_basicsize); 
+                            (long)s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -53,7 +53,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyByteArray_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -84,7 +83,6 @@
             ("mutable", "METH_NOARGS",
              """
                 PyObject *base;
-                char * p_str;
                 base = PyByteArray_FromStringAndSize("test", 10);
                 if (PyByteArray_GET_SIZE(base) != 10)
                     return PyLong_FromLong(-PyByteArray_GET_SIZE(base));
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -72,7 +72,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyBytes_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -100,7 +99,6 @@
                 PyObject *base;
                 PyTypeObject * type;
                 PyObject *obj;
-                char * p_str;
                 base = PyBytes_FromString("test");
                 if (PyBytes_GET_SIZE(base) != 4)
                     return PyLong_FromLong(-PyBytes_GET_SIZE(base));
@@ -258,7 +256,6 @@
         assert ref.c_ob_refcnt == prev_refcnt
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
-
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
         api.Py_DecRef(ref)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,8 +7,6 @@
 from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import platform
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -18,20 +16,7 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-def setup_module(module):
-    if os.name == 'nt':
-        # Do not open dreaded dialog box on segfault
-        import ctypes
-        SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
-        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
-        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
-        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
-        module.old_err_mode = old_err_mode
-
-def teardown_module(module):
-    if os.name == 'nt':
-        import ctypes
-        ctypes.windll.kernel32.SetErrorMode(module.old_err_mode)
+from .support import c_compile
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -46,7 +31,30 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
-def compile_extension_module(space, modname, include_dirs=[], **kwds):
+def convert_sources_to_files(sources, dirname):
+    files = []
+    for i, source in enumerate(sources):
+        filename = dirname / ('source_%d.c' % i)
+        with filename.open('w') as f:
+            f.write(str(source))
+        files.append(filename)
+    return files
+
+def create_so(modname, include_dirs, source_strings=None, source_files=None,
+        compile_extra=None, link_extra=None, libraries=None):
+    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+    if source_strings:
+        assert not source_files
+        files = convert_sources_to_files(source_strings, dirname)
+        source_files = files
+    soname = c_compile(source_files, outputfilename=str(dirname/modname),
+        compile_extra=compile_extra, link_extra=link_extra,
+        include_dirs=include_dirs,
+        libraries=libraries)
+    return soname
+
+def compile_extension_module(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -60,38 +68,36 @@
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        libraries = [api_library]
         # '%s' undefined; assuming extern returning int
-        kwds["compile_extra"] = ["/we4013"]
+        compile_extra = ["/we4013"]
         # prevent linking with PythonXX.lib
         w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
                               (space.int_w(w_maj), space.int_w(w_min))]
-    elif sys.platform == 'darwin':
-        kwds["link_files"] = [str(api_library + '.dylib')]
     else:
-        kwds["link_files"] = [str(api_library + '.so')]
+        libraries = []
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
-            kwds["link_extra"]=["-g"]
+            compile_extra = ["-Werror", "-g", "-O0", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs=api.include_dirs + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=api.include_dirs + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra,
+            libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
     return str(pydname)
 
-def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds):
+def compile_extension_module_applevel(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -103,24 +109,23 @@
     build the module (so specify your source with one of those).
     """
     if sys.platform == 'win32':
-        kwds["compile_extra"] = ["/we4013"]
-        kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
+        compile_extra = ["/we4013"]
+        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
     elif sys.platform == 'darwin':
+        compile_extra = link_extra = None
         pass
     elif sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"]
+        compile_extra = [
+            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
+        link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs = [space.include_dir] + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=[space.include_dir] + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra)
     return str(soname)
 
 def freeze_refcnts(self):
@@ -286,8 +291,8 @@
                 separate_module_sources = []
             pydname = self.compile_extension_module(
                 space, name,
-                separate_module_files=separate_module_files,
-                separate_module_sources=separate_module_sources)
+                source_files=separate_module_files,
+                source_strings=separate_module_sources)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -330,16 +335,16 @@
                 """ % dict(name=name, init=init, body=body,
                            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
                                             if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(separate_module_sources=[code])
+                kwds = dict(source_strings=[code])
             else:
                 assert not PY_SSIZE_T_CLEAN
                 if filename is None:
                     filename = name
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(separate_module_files=[filename])
-            kwds['include_dirs'] = include_dirs
-            mod = self.compile_extension_module(space, name, **kwds)
+                kwds = dict(source_files=[filename])
+            mod = self.compile_extension_module(space, name,
+                    include_dirs=include_dirs, **kwds)
 
             if load_it:
                 if self.runappdirect:
@@ -995,7 +1000,7 @@
             ('bar', 'METH_NOARGS',
              '''
              /* reuse a name that is #defined in structmember.h */
-             int RO;
+             int RO = 0; (void)RO;
              Py_RETURN_NONE;
              '''
              ),
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py
--- a/pypy/module/cpyext/test/test_frameobject.py
+++ b/pypy/module/cpyext/test/test_frameobject.py
@@ -13,7 +13,7 @@
                  PyObject *empty_bytes = PyBytes_FromString("");
                  PyObject *empty_tuple = PyTuple_New(0);
                  PyCodeObject *py_code;
-                 PyFrameObject *py_frame;
+                 PyFrameObject *py_frame = NULL;
 
                  py_code = PyCode_New(
                      0,            /*int argcount,*/
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -239,7 +239,7 @@
             except:
                 skip('numpy not importable')
         else:
-            numpy_incl = os.path.abspath(os.path.dirname(__file__) + 
+            numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
             assert os.path.exists(numpy_incl)
             cls.w_numpy_include = cls.space.wrap([numpy_incl])
@@ -302,7 +302,7 @@
                 ),
                 ("test_DescrFromType", "METH_O",
                 """
-                    Signed typenum = PyLong_AsLong(args);
+                    long typenum = PyInt_AsLong(args);
                     return PyArray_DescrFromType(typenum);
                 """
                 ),
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -401,7 +401,6 @@
                  """
     Py_buffer buf;
     PyObject *str = PyBytes_FromString("hello, world.");
-    PyObject *result;
 
     if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                           1, PyBUF_WRITABLE)) {
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -130,7 +130,7 @@
         module = self.import_extension('foo', [
             ("run", "METH_NOARGS",
              """
-                long prev, next;
+                long prev;
                 PyObject *t = PyTuple_New(1);
                 prev = Py_True->ob_refcnt;
                 Py_INCREF(Py_True);
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -716,7 +716,6 @@
              """
                 IntLikeObject *intObj;
                 long intval;
-                PyObject *name;
 
                 if (!PyArg_ParseTuple(args, "i", &intval))
                     return NULL;
@@ -1037,7 +1036,6 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                PyObject *obj;
                 FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
                 FooType_Type.tp_base = &PyType_Type;
                 if (PyType_Ready(&FooType_Type) < 0) return NULL;
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -35,7 +35,7 @@
             ("test_GetSize_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyUnicode_GetSize(f);
+                 PyUnicode_GetSize(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -57,7 +57,6 @@
              """
                  PyObject *s, *t;
                  Py_UNICODE* c;
-                 Py_ssize_t len;
 
                  s = PyUnicode_FromUnicode(NULL, 4);
                  if (s == NULL)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -23,7 +23,7 @@
             guard_true(i14, descr=...)
             guard_not_invalidated(descr=...)
             i16 = int_eq(i6, %d)
-            i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=)
+            i19 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i6, i10, descr=)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
             i22 = int_ge(i19, i10)
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -3,10 +3,12 @@
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--options] pypy-VER-PLATFORM
+    package.py [--options] --archive-name=pypy-VER-PLATFORM
 
 The output is found in the directory from --builddir,
 by default /tmp/usession-YOURNAME/build/.
+
+For a list of all options, see 'package.py --help'.
 """
 
 import shutil
@@ -64,6 +66,7 @@
     name = options.name
     if not name:
         name = 'pypy-nightly'
+    assert '/' not in name
     rename_pypy_c = options.pypy_c
     override_pypy_c = options.override_pypy_c
 
@@ -299,26 +302,12 @@
         help='destination dir for archive')
     parser.add_argument('--override_pypy_c', type=str, default='',
         help='use as pypy exe instead of pypy/goal/pypy-c')
-    # Positional arguments, for backward compatability with buldbots
-    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
-        metavar='[archive-name] [rename_pypy_c] [targetdir] [override_pypy_c]',
-        )
     options = parser.parse_args(args)
 
-    # Handle positional arguments, choke if both methods are used
-    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
-                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
-        if len(options.extra_args)>i:
-            if getattr(options, target) != default:
-                print 'positional argument',i,target,'already has value',getattr(options, target)
-                parser.print_help()
-                return
-            setattr(options, target, options.extra_args[i])
     if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
         options.nostrip = True
-
     if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        options.tk = True
+        options.no_tk = True
     if not options.builddir:
         # The import actually creates the udir directory
         from rpython.tool.udir import udir
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -21,8 +21,10 @@
 
     def test_dir_structure(self, test='test'):
         retval, builddir = package.package(
-            '--without-cffi', str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--without-cffi',
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
         assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d' % CPYTHON_VERSION[0]
@@ -71,8 +73,9 @@
         builddir = udir.ensure("build", dir=True)
         retval, builddir = package.package(
             '--without-cffi', '--builddir', str(builddir),
-            str(py.path.local(pypydir).dirpath()),
-            test, self.rename_pypy_c, _fake=True)
+            '--archive-name', test,
+            '--rename_pypy_c', self.rename_pypy_c,
+            _fake=True)
 
     def test_with_zipfile_module(self):
         prev = package.USE_ZIPFILE_MODULE
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -324,17 +324,19 @@
         def check(frame):
             expected_size = 1
             idx = 0
+            fixed_size = self.cpu.JITFRAME_FIXED_SIZE
             if self.cpu.backend_name.startswith('arm'):
                 # jitframe fixed part is larger here
                 expected_size = 2
                 idx = 1
+                fixed_size -= 32
             assert len(frame.jf_gcmap) == expected_size
-            if self.cpu.IS_64_BIT:
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE + 1  # +1 from i0
-            else:
-                assert frame.jf_gcmap[idx]
-                exp_idx = self.cpu.JITFRAME_FIXED_SIZE - 32 * idx + 1 # +1 from i0
-            assert frame.jf_gcmap[idx] == (1 << (exp_idx + 1)) | (1 << exp_idx)
+            # check that we have two bits set, and that they are in two
+            # registers (p0 and p1 are moved away when doing p2, but not
+            # spilled, just moved to different registers)
+            bits = [n for n in range(fixed_size)
+                      if frame.jf_gcmap[idx] & (1<> (LONG_BIT - 1)) & (p != x))
 
+def _ll_2_int_mod(x, y):
+    # same comments as _ll_2_int_floordiv()
+    r = x % y
+    # the JIT knows that if x and y are both positive, this doesn't change 'r'
+    r -= y & (((x ^ y) & (r | -r)) >> (LONG_BIT - 1))
+    return r
+
 
 def _ll_1_cast_uint_to_float(x):
     # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
@@ -431,6 +438,7 @@
 inline_calls_to = [
     ('int_abs',              [lltype.Signed],                lltype.Signed),
     ('int_floordiv',         [lltype.Signed, lltype.Signed], lltype.Signed),
+    ('int_mod',              [lltype.Signed, lltype.Signed], lltype.Signed),
     ('ll_math.ll_math_sqrt', [lltype.Float],                 lltype.Float),
 ]
 
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -478,7 +478,7 @@
             except ZeroDivisionError:
                 return -42
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_floordiv_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_div_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
@@ -505,7 +505,7 @@
                 return 42
         # XXX so far, this really produces a int_mod_ovf_zer...
         self.encoding_test(f, [7, 2], """
-            residual_call_ir_i $<* fn ll_int_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
+            residual_call_ir_i $<* fn ll_int_py_mod_ovf_zer__Signed_Signed>, I[%i0, %i1], R[],  -> %i2
             -live-
             catch_exception L1
             int_return %i2
diff --git a/rpython/jit/codewriter/test/test_support.py b/rpython/jit/codewriter/test/test_support.py
--- a/rpython/jit/codewriter/test/test_support.py
+++ b/rpython/jit/codewriter/test/test_support.py
@@ -144,11 +144,13 @@
     assert _ll_1_int_abs(-10) == 10
     assert _ll_1_int_abs(-sys.maxint) == sys.maxint
 
-def test_int_floordiv():
+def test_int_floordiv_mod():
     from rpython.rtyper.lltypesystem.lloperation import llop
-    from rpython.jit.codewriter.support import _ll_2_int_floordiv
+    from rpython.jit.codewriter.support import _ll_2_int_floordiv, _ll_2_int_mod
     for x in range(-6, 7):
         for y in range(-3, 4):
             if y != 0:
                 assert (_ll_2_int_floordiv(x, y) ==
                         llop.int_floordiv(lltype.Signed, x, y))
+                assert (_ll_2_int_mod(x, y) ==
+                        llop.int_mod(lltype.Signed, x, y))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -97,17 +97,14 @@
         self.emit_operation(op)
 
         r = self.getintbound(op)
-        if b2.is_constant():
-            val = b2.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.is_constant():
-            val = b1.lower
-            if val >= 0:
-                r.intersect(IntBound(0, val))
-        elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)):
-            lesser = min(b1.upper, b2.upper)
-            r.intersect(IntBound(0, next_pow2_m1(lesser)))
+        pos1 = b1.known_ge(IntBound(0, 0))
+        pos2 = b2.known_ge(IntBound(0, 0))
+        if pos1 or pos2:
+            r.make_ge(IntBound(0, 0))
+        if pos1:
+            r.make_le(b1)
+        if pos2:
+            r.make_le(b2)
 
     def optimize_INT_SUB(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5188,6 +5188,25 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_int_and_positive(self):
+        ops = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        i4 = int_ge(i3, 0)
+        guard_true(i4) []
+        jump(i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_ge(i1, 0)
+        guard_true(i2) []
+        i3 = int_and(i0, i1)
+        jump(i3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_or_cmp_above_bounds(self):
         ops = """
         [p0,p1]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -972,6 +972,58 @@
         assert res == expected
         # should contain a call_i(..., OS=OS_INT_PY_DIV)
 
+    def test_int_c_mod(self):
+        from rpython.rlib.rarithmetic import int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                t += int_c_mod(-100, i)
+                i += 1
+            return t
+        expected = -sum([100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # should contain a call_i(..., OS=OS_INT_PY_MOD)
+
+    def test_positive_c_div_mod(self):
+        from rpython.rlib.rarithmetic import int_c_div, int_c_mod
+        myjitdriver = JitDriver(greens = [], reds = ['i', 't'])
+        def f(i):
+            t = 0
+            while i < 10:
+                myjitdriver.can_enter_jit(i=i, t=t)
+                myjitdriver.jit_merge_point(i=i, t=t)
+                assert i > 0
+                t += int_c_div(100, i) - int_c_mod(100, i)
+                i += 1
+            return t
+        expected = sum([100 // n - 100 % n for n in range(1, 10)])
+        assert f(1) == expected
+        res = self.meta_interp(f, [1])
+        assert res == expected
+        # all the correction code should be dead now, xxx test that
+
+    def test_int_c_div_by_constant(self):
+        from rpython.rlib.rarithmetic import int_c_div
+        myjitdriver = JitDriver(greens = ['k'], reds = ['i', 't'])
+        def f(i, k):
+            t = 0
+            while i < 100:
+                myjitdriver.can_enter_jit(i=i, t=t, k=k)
+                myjitdriver.jit_merge_point(i=i, t=t, k=k)
+                t += int_c_div(i, k)
+                i += 1
+            return t
+        expected = sum([i // 10 for i in range(51, 100)])
+        assert f(-50, 10) == expected
+        res = self.meta_interp(f, [-50, 10])
+        assert res == expected
+        self.check_resops(call=0, uint_mul_high=2)
+
     def test_float(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res'])
         def f(x, y):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -281,11 +281,12 @@
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
+        "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
-        
+
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
@@ -644,6 +645,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -703,6 +705,7 @@
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
             result = self.nursery_free
+            ll_assert(result != llmemory.NULL, "uninitialized nursery")
             self.nursery_free = new_free = result + totalsize
             if new_free > self.nursery_top:
                 result = self.collect_and_reserve(totalsize)
@@ -1139,7 +1142,8 @@
         Implemented a bit obscurely by checking an unrelated flag
         that can never be set on a young object -- except if tid == -42.
         """
-        assert self.is_in_nursery(obj)
+        ll_assert(self.is_in_nursery(obj),
+                  "Can't forward an object outside the nursery.")
         tid = self.header(obj).tid
         result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
         if result:
@@ -1463,7 +1467,8 @@
                 objhdr.tid |= GCFLAG_CARDS_SET
 
         remember_young_pointer_from_array2._dont_inline_ = True
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         self.remember_young_pointer_from_array2 = (
             remember_young_pointer_from_array2)
 
@@ -1513,7 +1518,8 @@
             return True
         # ^^^ a fast path of write-barrier
         #
-        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+        if (self.card_page_indices > 0 and     # check constant-folded
+            source_hdr.tid & GCFLAG_HAS_CARDS != 0):
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1548,7 +1554,8 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
-        assert self.card_page_indices > 0
+        ll_assert(self.card_page_indices > 0,
+                  "non-positive card_page_indices")
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1721,12 +1728,15 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         self.surviving_pinned_objects.sort()
-        assert self.pinned_objects_in_nursery == \
-            self.surviving_pinned_objects.length()
+        ll_assert(
+            self.pinned_objects_in_nursery == \
+            self.surviving_pinned_objects.length(),
+            "pinned_objects_in_nursery != surviving_pinned_objects.length()")
         while self.surviving_pinned_objects.non_empty():
             #
             cur = self.surviving_pinned_objects.pop()
-            assert cur >= prev
+            ll_assert(
+                cur >= prev, "pinned objects encountered in backwards order")
             #
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
@@ -1784,7 +1794,8 @@
         debug_stop("gc-minor")
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
-        assert self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN
+        ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
+                  "!GCFLAG_PINNED_OBJECT_PARENT_KNOWN, but requested to reset.")
         self.header(obj).tid &= ~GCFLAG_PINNED_OBJECT_PARENT_KNOWN
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -554,6 +554,7 @@
         assert res # we optimized it
         assert hdr_dst.tid & minimark.GCFLAG_TRACK_YOUNG_PTRS == 0 # and we copied the flag
         #
+        self.gc.card_page_indices = 128     # force > 0
         hdr_src.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_dst.tid |= minimark.GCFLAG_TRACK_YOUNG_PTRS
         hdr_src.tid |= minimark.GCFLAG_HAS_CARDS
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -148,7 +148,8 @@
                                                  ('elements', FFI_TYPE_PP)])
 
     ffi_cif = rffi_platform.Struct('ffi_cif', [])
-    ffi_closure = rffi_platform.Struct('ffi_closure', [])
+    ffi_closure = rffi_platform.Struct('ffi_closure',
+                                       [('user_data', rffi.VOIDP)])
 
 def add_simple_type(type_name):
     for name in ['size', 'alignment', 'type']:
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -662,6 +662,14 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_floordiv(lltype.Signed, x, y)
 
+def int_c_mod(x, y):
+    """Return the result of the C-style 'x % y'.  This differs from the
+    Python-style division if (x < 0  xor y < 0).
+    """
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    return llop.int_mod(lltype.Signed, x, y)
+
 @objectmodel.specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -1,10 +1,17 @@
-#define HAVE_SYS_UCONTEXT_H
+#if !defined(__OpenBSD__)
+#  define HAVE_SYS_UCONTEXT_H
+#else
+#  define HAVE_SIGNAL_H
+#endif
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
   #ifdef __i386__
     #define PC_FROM_UCONTEXT uc_mcontext.mc_eip
   #else
     #define PC_FROM_UCONTEXT uc_mcontext.mc_rip
   #endif
+#elif defined(__OpenBSD__)
+#define PC_FROM_UCONTEXT sc_rip
 #elif defined( __APPLE__)
   #if ((ULONG_MAX) == (UINT_MAX))
     #define PC_FROM_UCONTEXT uc_mcontext->__ss.__eip
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -65,6 +65,10 @@
 #elif defined(HAVE_CYGWIN_SIGNAL_H)
 #include 
 typedef ucontext ucontext_t;
+#elif defined(HAVE_SIGNAL_H)
+#include 
+#else
+#  error "don't know how to get the pc on this platform"
 #endif
 
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -401,10 +401,13 @@
 
 @given(strategies.integers(min_value=0, max_value=sys.maxint),
        strategies.integers(min_value=1, max_value=sys.maxint))
-def test_int_c_div(x, y):
+def test_int_c_div_mod(x, y):
     assert int_c_div(~x, y) == -(abs(~x) // y)
     assert int_c_div( x,-y) == -(x // y)
     assert int_c_div(~x,-y) == +(abs(~x) // y)
+    for x1 in [x, ~x]:
+        for y1 in [y, -y]:
+            assert int_c_div(x1, y1) * y1 + int_c_mod(x1, y1) == x1
 
 # these can't be prebuilt on 32bit
 U1 = r_ulonglong(0x0102030405060708L)
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -236,11 +236,11 @@
         return _rtype_template(hop, 'mul_ovf')
 
     def rtype_floordiv(_, hop):
-        return _rtype_call_helper(hop, 'floordiv', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div', [ZeroDivisionError])
     rtype_inplace_floordiv = rtype_floordiv
 
     def rtype_floordiv_ovf(_, hop):
-        return _rtype_call_helper(hop, 'floordiv_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_div_ovf', [ZeroDivisionError])
 
     # turn 'div' on integers into 'floordiv'
     rtype_div         = rtype_floordiv
@@ -250,11 +250,11 @@
     # 'def rtype_truediv' is delegated to the superclass FloatRepr
 
     def rtype_mod(_, hop):
-        return _rtype_call_helper(hop, 'mod', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod', [ZeroDivisionError])
     rtype_inplace_mod = rtype_mod
 
     def rtype_mod_ovf(_, hop):
-        return _rtype_call_helper(hop, 'mod_ovf', [ZeroDivisionError])
+        return _rtype_call_helper(hop, 'py_mod_ovf', [ZeroDivisionError])
 
     def rtype_xor(_, hop):
         return _rtype_template(hop, 'xor')
@@ -319,7 +319,7 @@
     vlist = hop.inputargs(repr, repr2)
     prefix = repr.opprefix
 
-    if '_ovf' in func or func.startswith(('mod', 'floordiv')):
+    if '_ovf' in func or func.startswith(('py_mod', 'py_div')):
         if prefix+func not in ('int_add_ovf', 'int_add_nonneg_ovf',
                                'int_sub_ovf', 'int_mul_ovf'):
             raise TyperError("%r should not be used here any more" % (func,))
@@ -353,7 +353,7 @@
             any_implicit_exception = True
 
     if not any_implicit_exception:
-        if not func.startswith(('mod', 'floordiv')):
+        if not func.startswith(('py_mod', 'py_div')):
             return _rtype_template(hop, func)
 
     repr = hop.r_result
@@ -388,7 +388,7 @@
 # ---------- floordiv ----------
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv(x, y):
+def ll_int_py_div(x, y):
     # Python, and RPython, assume that integer division truncates
     # towards -infinity.  However, in C, integer division truncates
     # towards 0.  So assuming that, we need to apply a correction
@@ -400,159 +400,159 @@
     return r + (u >> INT_BITS_1)
 
 @jit.oopspec("int.py_div(x, y)")
-def ll_int_floordiv_nonnegargs(x, y):
+def ll_int_py_div_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_floordiv(Signed, x, y)            # <= truncates like in C
-    ll_assert(r >= 0, "int_floordiv_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_div_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_floordiv_zer(x, y):
+def ll_int_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf(x, y):
+def ll_int_py_div_ovf(x, y):
     # JIT: intentionally not short-circuited to produce only one guard
     # and to remove the check fully if one of the arguments is known
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError("integer division")
-    return ll_int_floordiv(x, y)
+    return ll_int_py_div(x, y)
 
-def ll_int_floordiv_ovf_zer(x, y):
+def ll_int_py_div_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("integer division")
-    return ll_int_floordiv_ovf(x, y)
+    return ll_int_py_div_ovf(x, y)
 
 @jit.oopspec("int.udiv(x, y)")
-def ll_uint_floordiv(x, y):
+def ll_uint_py_div(x, y):
     return llop.uint_floordiv(Unsigned, x, y)
 
-def ll_uint_floordiv_zer(x, y):
+def ll_uint_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("unsigned integer division")
-    return ll_uint_floordiv(x, y)
+    return ll_uint_py_div(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_floordiv      = ll_int_floordiv
-    ll_llong_floordiv_zer  = ll_int_floordiv_zer
-    ll_ullong_floordiv     = ll_uint_floordiv
-    ll_ullong_floordiv_zer = ll_uint_floordiv_zer
+    ll_llong_py_div      = ll_int_py_div
+    ll_llong_py_div_zer  = ll_int_py_div_zer
+    ll_ullong_py_div     = ll_uint_py_div
+    ll_ullong_py_div_zer = ll_uint_py_div_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_floordiv(x, y):
+    def ll_llong_py_div(x, y):
         r = llop.llong_floordiv(SignedLongLong, x, y)  # <= truncates like in C
         p = r * y
         if y < 0: u = p - x
         else:     u = x - p
         return r + (u >> LLONG_BITS_1)
 
-    def ll_llong_floordiv_zer(x, y):
+    def ll_llong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("longlong division")
-        return ll_llong_floordiv(x, y)
+        return ll_llong_py_div(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_floordiv(x, y):
+    def ll_ullong_py_div(x, y):
         return llop.ullong_floordiv(UnsignedLongLong, x, y)
 
-    def ll_ullong_floordiv_zer(x, y):
+    def ll_ullong_py_div_zer(x, y):
         if y == 0:
             raise ZeroDivisionError("unsigned longlong division")
-        return ll_ullong_floordiv(x, y)
+        return ll_ullong_py_div(x, y)
 
 @jit.dont_look_inside
-def ll_lllong_floordiv(x, y):
+def ll_lllong_py_div(x, y):
     r = llop.lllong_floordiv(SignedLongLongLong, x, y) # <= truncates like in C
     p = r * y
     if y < 0: u = p - x
     else:     u = x - p
     return r + (u >> LLLONG_BITS_1)
 
-def ll_lllong_floordiv_zer(x, y):
+def ll_lllong_py_div_zer(x, y):
     if y == 0:
         raise ZeroDivisionError("longlonglong division")
-    return ll_lllong_floordiv(x, y)
+    return ll_lllong_py_div(x, y)
 
 
 # ---------- mod ----------
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod(x, y):
+def ll_int_py_mod(x, y):
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> INT_BITS_1))
 
 @jit.oopspec("int.py_mod(x, y)")
-def ll_int_mod_nonnegargs(x, y):
+def ll_int_py_mod_nonnegargs(x, y):
     from rpython.rlib.debug import ll_assert
     r = llop.int_mod(Signed, x, y)                 # <= truncates like in C
-    ll_assert(r >= 0, "int_mod_nonnegargs(): one arg is negative")
+    ll_assert(r >= 0, "int_py_mod_nonnegargs(): one arg is negative")
     return r
 
-def ll_int_mod_zer(x, y):
+def ll_int_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf(x, y):
-    # see comment in ll_int_floordiv_ovf
+def ll_int_py_mod_ovf(x, y):
+    # see comment in ll_int_py_div_ovf
     if (x == -sys.maxint - 1) & (y == -1):
         raise OverflowError
-    return ll_int_mod(x, y)
+    return ll_int_py_mod(x, y)
 
-def ll_int_mod_ovf_zer(x, y):
+def ll_int_py_mod_ovf_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_int_mod_ovf(x, y)
+    return ll_int_py_mod_ovf(x, y)
 
 @jit.oopspec("int.umod(x, y)")
-def ll_uint_mod(x, y):
+def ll_uint_py_mod(x, y):
     return llop.uint_mod(Unsigned, x, y)
 
-def ll_uint_mod_zer(x, y):
+def ll_uint_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_uint_mod(x, y)
+    return ll_uint_py_mod(x, y)
 
 if SignedLongLong == Signed:
-    ll_llong_mod      = ll_int_mod
-    ll_llong_mod_zer  = ll_int_mod_zer
-    ll_ullong_mod     = ll_uint_mod
-    ll_ullong_mod_zer = ll_uint_mod_zer
+    ll_llong_py_mod      = ll_int_py_mod
+    ll_llong_py_mod_zer  = ll_int_py_mod_zer
+    ll_ullong_py_mod     = ll_uint_py_mod
+    ll_ullong_py_mod_zer = ll_uint_py_mod_zer
 else:
     @jit.dont_look_inside
-    def ll_llong_mod(x, y):
+    def ll_llong_py_mod(x, y):
         r = llop.llong_mod(SignedLongLong, x, y)    # <= truncates like in C
         if y < 0: u = -r
         else:     u = r
         return r + (y & (u >> LLONG_BITS_1))
 
-    def ll_llong_mod_zer(x, y):
+    def ll_llong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return ll_llong_mod(x, y)
+        return ll_llong_py_mod(x, y)
 
     @jit.dont_look_inside
-    def ll_ullong_mod(x, y):
+    def ll_ullong_py_mod(x, y):
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
-    def ll_ullong_mod_zer(x, y):
+    def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
         return llop.ullong_mod(UnsignedLongLong, x, y)
 
 @jit.dont_look_inside
-def ll_lllong_mod(x, y):
+def ll_lllong_py_mod(x, y):
     r = llop.lllong_mod(SignedLongLongLong, x, y)  # <= truncates like in C
     if y < 0: u = -r
     else:     u = r
     return r + (y & (u >> LLLONG_BITS_1))
 
-def ll_lllong_mod_zer(x, y):
+def ll_lllong_py_mod_zer(x, y):
     if y == 0:
         raise ZeroDivisionError
-    return ll_lllong_mod(x, y)
+    return ll_lllong_py_mod(x, y)
 
 
 # ---------- lshift, neg, abs ----------
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -390,7 +390,7 @@
         res = self.interpret(f, [sys.maxint])
         assert res == 0
 
-    def test_int_floordiv_nonnegargs(self):
+    def test_int_py_div_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
@@ -398,7 +398,7 @@
         res = self.interpret(f, [1234567, 123])
         assert res == 1234567 // 123
 
-    def test_int_mod_nonnegargs(self):
+    def test_int_py_mod_nonnegargs(self):
         def f(x, y):
             assert x >= 0
             assert y >= 0
diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py
--- a/rpython/translator/sandbox/sandlib.py
+++ b/rpython/translator/sandbox/sandlib.py
@@ -530,6 +530,9 @@
     def do_ll_os__ll_os_unlink(self, vpathname):
         raise OSError(errno.EPERM, "write access denied")
 
+    def do_ll_os__ll_os_mkdir(self, vpathname, mode=None):
+        raise OSError(errno.EPERM, "write access denied")
+
     def do_ll_os__ll_os_getuid(self):
         return UID
     do_ll_os__ll_os_geteuid = do_ll_os__ll_os_getuid
diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py
--- a/rpython/translator/test/test_simplify.py
+++ b/rpython/translator/test/test_simplify.py
@@ -55,7 +55,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \
@@ -73,7 +73,7 @@
     graph, _ = translate(f, [int, int], backend_optimize=False)
     assert len(graph.startblock.operations) == 1
     assert graph.startblock.operations[0].opname == 'direct_call'
-    assert 'int_floordiv_ovf_zer' in repr(
+    assert 'int_py_div_ovf_zer' in repr(
         graph.startblock.operations[0].args[0].value)
     assert len(graph.startblock.exits) == 3
     assert [link.target.operations for link in graph.startblock.exits[1:]] == \

From pypy.commits at gmail.com  Mon Jun 20 15:19:24 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 12:19:24 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: hg merge py3k
Message-ID: <576841bc.441ac20a.67e79.3c34@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85262:2713931662a9
Date: 2016-06-20 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/2713931662a9/

Log:	hg merge py3k

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,9 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -22,3 +25,9 @@
 
 .. branch: incminimark-ll_assert
 .. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1524,7 +1524,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test/test_structseq.py
--- a/pypy/module/cpyext/test/test_structseq.py
+++ b/pypy/module/cpyext/test/test_structseq.py
@@ -6,6 +6,7 @@
 
 class AppTestStructSeq(AppTestCpythonExtensionBase):
     def test_StructSeq(self):
+        skip("XXX: https://bugs.pypy.org/issue1557")
         module = self.import_extension('foo',
         prologue="""
             #include 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rdynload
 import sys
 
 _WIN = sys.platform == 'win32'
@@ -18,6 +19,7 @@
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
+        self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))',
@@ -79,6 +81,8 @@
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'getdlopenflags'        : 'system.getdlopenflags',
+        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -108,3 +108,9 @@
     ]
     w_thread_info = app.wget(space, "thread_info")
     return space.call_function(w_thread_info, space.newtuple(info_w))
+
+def getdlopenflags(space):
+    return space.wrap(space.sys.dlopenflags)
+
+def setdlopenflags(space, w_flags):
+    space.sys.dlopenflags = space.int_w(w_flags)
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,14 +478,12 @@
 
     def test_dlopenflags(self):
         import sys
-        if hasattr(sys, "setdlopenflags"):
-            assert hasattr(sys, "getdlopenflags")
-            raises(TypeError, sys.getdlopenflags, 42)
-            oldflags = sys.getdlopenflags()
-            raises(TypeError, sys.setdlopenflags)
-            sys.setdlopenflags(oldflags+1)
-            assert sys.getdlopenflags() == oldflags+1
-            sys.setdlopenflags(oldflags)
+        raises(TypeError, sys.getdlopenflags, 42)
+        oldflags = sys.getdlopenflags()
+        raises(TypeError, sys.setdlopenflags)
+        sys.setdlopenflags(oldflags+1)
+        assert sys.getdlopenflags() == oldflags+1
+        sys.setdlopenflags(oldflags)
 
     def test_refcount(self):
         import sys
@@ -661,7 +659,7 @@
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
-        
+
         class Tracer:
             def __init__(self):
                 self.events = []
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -427,11 +427,11 @@
 class FakeModule(W_Root):
     def __init__(self):
         self.w_dict = w_some_obj()
-
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
+FakeObjSpace.sys.dlopenflags = 123
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -181,6 +181,7 @@
             return space.newint(value.count(sub, start, end))
         else:
             res = count(value, sub, start, end)
+            assert res >= 0
             return space.wrap(max(res, 0))
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -218,6 +218,7 @@
 
         check(bytearray(b'abc').replace(b'b', bytearray(b'd')), b'adc')
         check(bytearray(b'abc').replace(b'b', b'd'), b'adc')
+        check(bytearray(b'').replace(b'a', b'ab'), b'')
 
         check(bytearray(b'abc').upper(), b'ABC')
         check(bytearray(b'ABC').lower(), b'abc')
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -26,7 +26,7 @@
 
 if _MAC_OS:
     pre_include_bits = ['#define MACOSX']
-else: 
+else:
     pre_include_bits = []
 
 if _FREEBSD or _NETBSD or _WIN32:
@@ -145,15 +145,20 @@
         else:
             return lltype.nullptr(rffi.VOIDP.TO)
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        mode = RTLD_NOW
+        if RTLD_LOCAL is not None:
+            mode |= RTLD_LOCAL
+        return mode
+
     def dlopen(name, mode=-1):
         """ Wrapper around C-level dlopen
         """
         if mode == -1:
-            if RTLD_LOCAL is not None:
-                mode = RTLD_LOCAL
-            else:
-                mode = 0
-        if (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
+            mode = _dlopen_default_mode()
+        elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
@@ -193,6 +198,11 @@
     DLLHANDLE = rwin32.HMODULE
     RTLD_GLOBAL = None
 
+    def _dlopen_default_mode():
+        """ The default dlopen mode if it hasn't been changed by the user.
+        """
+        return 0
+
     def dlopen(name, mode=-1):
         # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -291,6 +291,7 @@
     return _search(value, other, start, end, SEARCH_COUNT)
 
 # -------------- substring searching helper ----------------
+# XXX a lot of code duplication with lltypesystem.rstr :-(
 
 SEARCH_COUNT = 0
 SEARCH_FIND = 1
@@ -309,6 +310,8 @@
     if end > len(value):
         end = len(value)
     if start > end:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     count = 0
@@ -326,6 +329,8 @@
     w = n - m
 
     if w < 0:
+        if mode == SEARCH_COUNT:
+            return 0
         return -1
 
     mlast = m - 1
@@ -570,18 +575,20 @@
 
 class ByteListBuilder(object):
     def __init__(self, init_size=INIT_SIZE):
+        assert init_size >= 0
         self.l = newlist_hint(init_size)
 
     @specialize.argtype(1)
     def append(self, s):
+        l = self.l
         for c in s:
-            self.l.append(c)
+            l.append(c)
 
     @specialize.argtype(1)
     def append_slice(self, s, start, end):
-        assert 0 <= start <= end <= len(s)
-        for c in s[start:end]:
-            self.l.append(c)
+        l = self.l
+        for i in xrange(start, end):
+            l.append(s[i])
 
     def append_multiple_char(self, c, times):
         assert isinstance(c, str)
@@ -589,8 +596,9 @@
 
     def append_charpsize(self, s, size):
         assert size >= 0
+        l = self.l
         for i in xrange(size):
-            self.l.append(s[i])
+            l.append(s[i])
 
     def build(self):
         return self.l
diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py
--- a/rpython/rlib/test/test_rstring.py
+++ b/rpython/rlib/test/test_rstring.py
@@ -231,6 +231,10 @@
     check_search(count, 'one two three', 'e', 0, 1, res=0)
     check_search(count, 'one two three', '', 0, 13, res=14)
 
+    check_search(count, '', 'ab', 0, 0, res=0)
+    check_search(count, 'a', 'ab', 0, 1, res=0)
+    check_search(count, 'ac', 'ab', 0, 2, res=0)
+
 
 class TestTranslates(BaseRtypingTest):
     def test_split_rsplit(self):
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -348,19 +348,30 @@
     _about_ = llhelper
 
     def compute_result_annotation(self, s_F, s_callable):
+        from rpython.annotator.description import FunctionDesc
         assert s_F.is_constant()
-        assert s_callable.is_constant()
+        assert isinstance(s_callable, annmodel.SomePBC)
         F = s_F.const
         FUNC = F.TO
         args_s = [lltype_to_annotation(T) for T in FUNC.ARGS]
-        key = (llhelper, s_callable.const)
-        s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
-        assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
+        for desc in s_callable.descriptions:
+            assert isinstance(desc, FunctionDesc)
+            assert desc.pyobj is not None
+            if s_callable.is_constant():
+                assert s_callable.const is desc.pyobj
+            key = (llhelper, desc.pyobj)
+            s_res = self.bookkeeper.emulate_pbc_call(key, s_callable, args_s)
+            assert lltype_to_annotation(FUNC.RESULT).contains(s_res)
         return SomePtr(F)
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
-        return hop.args_r[1].get_unique_llfn()
+        if hop.args_s[1].is_constant():
+            return hop.args_r[1].get_unique_llfn()
+        else:
+            F = hop.args_s[0].const
+            assert hop.args_r[1].lowleveltype == F
+            return hop.inputarg(hop.args_r[1], 1)
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -414,7 +414,7 @@
         if self.s_pbc.can_be_None:
             self.descriptions.insert(0, None)
         POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
-                              hints={'nolength': True})
+                              hints={'nolength': True, 'immutable': True})
         pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
                                immortal=True)
         for i, desc in enumerate(self.descriptions):
@@ -564,7 +564,7 @@
     if r_to in r_from._conversion_tables:
         return r_from._conversion_tables[r_to]
     else:
-        t = malloc(Array(Char, hints={'nolength': True}),
+        t = malloc(Array(Char, hints={'nolength': True, 'immutable': True}),
                    len(r_from.descriptions), immortal=True)
         l = []
         for i, d in enumerate(r_from.descriptions):
@@ -577,7 +577,7 @@
         if l == range(len(r_from.descriptions)):
             r = None
         else:
-            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
+            r = inputconst(typeOf(t), t)
         r_from._conversion_tables[r_to] = r
         return r
 
diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -462,6 +462,30 @@
     res = interpret(h, [8, 5, 2])
     assert res == 99
 
+def test_llhelper_multiple_functions():
+    S = GcStruct('S', ('x', Signed), ('y', Signed))
+    def f(s):
+        return s.x - s.y
+    def g(s):
+        return s.x + s.y
+
+    F = Ptr(FuncType([Ptr(S)], Signed))
+
+    myfuncs = [f, g]
+
+    def h(x, y, z):
+        s = malloc(S)
+        s.x = x
+        s.y = y
+        fptr = llhelper(F, myfuncs[z])
+        assert typeOf(fptr) == F
+        return fptr(s)
+
+    res = interpret(h, [80, 5, 0])
+    assert res == 75
+    res = interpret(h, [80, 5, 1])
+    assert res == 85
+
 
 def test_cast_instance_to_base_ptr():
     class A:
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -2012,6 +2012,36 @@
         e = py.test.raises(ValueError, self.interpret, f, [3])
         assert str(e.value).startswith(r"exit case '\xff' not found")
 
+    @py.test.mark.parametrize('limit', [3, 5])
+    def test_conversion_table(self, limit):
+        # with limit==3, check conversion from Char to Ptr(Func).
+        # with limit>3, check conversion from Char to Char.
+        def f1():
+            return 111
+        def f2():
+            return 222
+        def f3():
+            return 333
+        def g(n):
+            if n & 1:
+                return f1
+            else:
+                return f2
+        def f(n):
+            x = g(n)    # can be f1 or f2
+            if n > 10:
+                x = f3  # now can be f1 or f2 or f3
+            return x()
+
+        from rpython.config.translationoption import get_combined_translation_config
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.withsmallfuncsets = limit
+        assert self.interpret(f, [3]) == 111
+        assert self.interpret(f, [4]) == 222
+        assert self.interpret(f, [13]) == 333
+        assert self.interpret(f, [14]) == 333
+
+
 def test_smallfuncsets_basic():
     from rpython.translator.translator import TranslationContext, graphof
     from rpython.config.translationoption import get_combined_translation_config
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -972,6 +972,13 @@
             s.count(s, -10)
         py.test.raises(AnnotatorError, self.interpret, f, ())
 
+    def test_count_in_empty_string(self):
+        const = self.const
+        def fn():
+            return const('').count(const('ab'))
+        res = self.interpret(fn, [])
+        assert res == 0
+
     def test_getitem_exc(self):
         const = self.const
         def f(x):

From pypy.commits at gmail.com  Mon Jun 20 17:11:42 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Mon, 20 Jun 2016 14:11:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: call non unpack methods temporarily
Message-ID: <57685c0e.4a79c20a.a7e6b.3328@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85263:ef3eebed1864
Date: 2016-06-20 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/ef3eebed1864/

Log:	call non unpack methods temporarily

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1331,37 +1331,40 @@
         self.pushvalue(w_set)
 
     def BUILD_SET_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newset()
-        for i in range(itemcount, 0, -1):
-            w_item = self.popvalue()
-            #self.space.peek(i)
-            self.space.call_method(w_sum, 'update', w_item)
-        #while itemcount != 0:
-        #    self.popvalue()
-        #    itemcount -= 1
-        self.pushvalue(w_sum)
+        self.BUILD_SET(itemcount, next_instr)
+        #w_sum = self.space.newset()
+        #for i in range(itemcount, 0, -1):
+        #    w_item = self.popvalue()
+        #    #self.space.peek(i)
+        #    self.space.call_method(w_sum, 'update', w_item)
+        ##while itemcount != 0:
+        ##    self.popvalue()
+        ##    itemcount -= 1
+        #self.pushvalue(w_sum)
 
     def BUILD_TUPLE_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newtuple()
-        for i in range(itemcount, 0, -1):
-            w_item = self.popvalue()
-            #self.space.peek(i)
-            self.space.call_method(w_sum, 'update', w_item)
-        #while itemcount != 0:
-        #    self.popvalue()
-        #    itemcount -= 1
-        self.pushvalue(w_sum)
+        self.BUILD_TUPLE(itemcount, next_instr)
+        #w_sum = self.space.newtuple()
+        #for i in range(itemcount, 0, -1):
+        #    w_item = self.popvalue()
+        #    #self.space.peek(i)
+        #    self.space.call_method(w_sum, 'update', w_item)
+        ##while itemcount != 0:
+        ##    self.popvalue()
+        ##    itemcount -= 1
+        #self.pushvalue(w_sum)
         
     def BUILD_LIST_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newlist()
-        for i in range(itemcount, 0, -1):
-            w_item = self.popvalue()
-            #self.space.peek(i)
-            self.space.call_method(w_sum, 'update', w_item)
-        #while itemcount != 0:
-        #    self.popvalue()
-        #    itemcount -= 1
-        self.pushvalue(w_sum)
+        self.BUILD_LIST(itemcount, next_instr)
+        #w_sum = self.space.newlist()
+        #for i in range(itemcount, 0, -1):
+        #    w_item = self.popvalue()
+        #    #self.space.peek(i)
+        #    self.space.call_method(w_sum, 'update', w_item)
+        ##while itemcount != 0:
+        ##    self.popvalue()
+        ##    itemcount -= 1
+        #self.pushvalue(w_sum)
         
     #TODO
     #get intersection, store as setentry
@@ -1379,15 +1382,16 @@
         self.pushvalue(w_sum)
         
     def BUILD_MAP_UNPACK(self, itemcount, next_instr):
-        w_sum = self.space.newdict()
-        for i in range(itemcount, 0, -1):
-            w_item = self.popvalue()
-            #self.space.peek(i)
-            self.space.call_method(w_sum, 'update', w_item)
-        #while itemcount != 0:
-        #    self.popvalue()
-        #    itemcount -= 1
-        self.pushvalue(w_sum)
+        self.BUILD_MAP(itemcount, next_instr)
+        #w_sum = self.space.newdict()
+        #for i in range(itemcount, 0, -1):
+        #    w_item = self.popvalue()
+        #    #self.space.peek(i)
+        #    self.space.call_method(w_sum, 'update', w_item)
+        ##while itemcount != 0:
+        ##    self.popvalue()
+        ##    itemcount -= 1
+        #self.pushvalue(w_sum)
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):

From pypy.commits at gmail.com  Mon Jun 20 17:39:27 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 20 Jun 2016 14:39:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <5768628f.69fac20a.5bcff.ffff8c3a@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r85264:d70f72636d06
Date: 2016-06-20 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d70f72636d06/

Log:	fix translation

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -478,7 +478,7 @@
             self.intbound = info
 
     def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
-                                optimizer):
+                                state):
         other_intbound = None
         if isinstance(other, NotVirtualStateInfoInt):
             other_intbound = other.intbound
@@ -490,7 +490,7 @@
             self.intbound.contains(runtime_box.getint())):
             # this may generate a few more guards than needed, but they are
             # optimized away when emitting them
-            self.intbound.make_guards(box, extra_guards, optimizer)
+            self.intbound.make_guards(box, extra_guards, state.optimizer)
             return
         raise VirtualStatesCantMatch("intbounds don't match")
 

From pypy.commits at gmail.com  Mon Jun 20 18:16:29 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Commit what I have so far so I can test on
 windows. Everything added might not work properly. (Tests certainly don't
 pass)
Message-ID: <57686b3d.2237c20a.c8387.ffffc58d@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85265:b00ce92dc54f
Date: 2016-05-17 00:07 -0400
http://bitbucket.org/pypy/pypy/changeset/b00ce92dc54f/

Log:	Commit what I have so far so I can test on windows. Everything added
	might not work properly. (Tests certainly don't pass)

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -417,7 +417,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True),
+    RegrTest('test_time.py', core=True, usemodules="struct"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -40,6 +40,7 @@
         'struct_time': 'app_time.struct_time',
         '__doc__': 'app_time.__doc__',
         'strptime': 'app_time.strptime',
+        'get_clock_info': 'app_time.get_clock_info'
     }
 
     def startup(self, space):
diff --git a/pypy/module/time/app_time.py b/pypy/module/time/app_time.py
--- a/pypy/module/time/app_time.py
+++ b/pypy/module/time/app_time.py
@@ -1,7 +1,8 @@
 # NOT_RPYTHON
 
 from _structseq import structseqtype, structseqfield
-
+from types import SimpleNamespace
+import time
 class struct_time(metaclass=structseqtype):
     __module__ = 'time'
     name = 'time.struct_time'
@@ -26,6 +27,28 @@
     import _strptime     # from the CPython standard library
     return _strptime._strptime_time(string, format)
 
+def get_clock_info(name):
+    info = SimpleNamespace()
+    info.implementation = ""
+    info.monotonic = 0
+    info.adjustable = 0
+    info.resolution = 1.0
+    print(id(info), "id in app")
+
+    if name == "time":
+        time.time(info)
+    elif name == "monotonic":
+        time.monotonic(info)
+    elif name == "clock":
+        time.clock(info)
+    elif name == "perf_counter":
+        time.perf_counter(info)
+    elif name == "process_time":
+        time.process_time(info)
+    else:
+        raise ValueError("unknown clock")
+    return info
+
 __doc__ = """This module provides various functions to manipulate time values.
 
 There are two standard representations of time.  One is the number
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -103,6 +103,14 @@
         def get_interrupt_event(self):
             return globalState.interrupt_event
 
+    # Can I just use one of the state classes above?
+    # I don't really get why an instance is better than a plain module
+    # attr, but following advice from armin
+    class TimeState(object):
+        def __init__(self):
+            self.n_overflow = 0
+            self.last_ticks = 0
+    time_state = TimeState()
 
 _includes = ["time.h"]
 if _POSIX:
@@ -118,6 +126,7 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
+    has_gettickcount64 = platform.Has("GetTickCount64")
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
@@ -185,6 +194,7 @@
 
 CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
 HAS_CLOCK_GETTIME = cConfig.has_clock_gettime
+HAS_GETTICKCOUNT64 = cConfig.has_gettickcount64
 clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
@@ -503,18 +513,19 @@
 
     Return the current time in seconds since the Epoch.
     Fractions of a second may be present if the system clock provides them."""
-
     secs = pytime.time()
     return space.wrap(secs)
 
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
+# TODO: Remember what this is for...
+def get_time_time_clock_info(space, w_info):
+    # Can't piggy back on time.time because time.time delegates to the 
+    # host python's time.time (so we can't see the internals)
+    if HAS_CLOCK_GETTIME:
+        try:
+            res = clock_getres(space, cConfig.CLOCK_REALTIME)
+        except OperationError:
+            res = 1e-9
+    #else: ???
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -717,9 +728,51 @@
 if _WIN:
     # untested so far
     _GetTickCount64 = rwin32.winexternal('GetTickCount64', [], rffi.ULONGLONG)
+    _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
+    LPDWORD = rwin32.LPDWORD
+    _GetSystemTimeAdjustment = rwin32.winexternal(
+                                            'GetSystemTimeAdjustment',
+                                            [LPDWORD, LPDWORD, rffi.INTP], 
+                                            rffi.INT)
 
-    def monotonic(space):
-        return space.wrap(_GetTickCount64() * 1e-3)
+    def monotonic(space, w_info=None):
+        result = 0
+        if HAS_GETTICKCOUNT64:
+            result = _GetTickCount64() * 1e-3
+        else:
+            ticks = _GetTickCount()
+            if ticks < time_state.last_ticks:
+                time_state.n_overflow += 1
+            time_state.last_ticks = ticks
+            result = math.ldexp(time_state.n_overflow, 32)
+            result = result + ticks
+            result = result * 1e-3
+            
+        if w_info is not None:
+            if HAS_GETTICKCOUNT64:
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("GetTickCount64()"))
+            else:
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("GetTickCount()"))
+            resolution = 1e-7
+            with lltype.scoped_alloc(rwin32.LPDWORD) as time_adjustment, \
+                 lltype.scoped_alloc(rwin32.LPDWORD) as time_increment, \
+                 lltype.scoped_alloc(rwin32.FILETIME) as is_time_adjustment_disabled:
+                ok = _GetSystemTimeAdjustment(time_adjustment,
+                                              time_increment,
+                                              is_time_adjustment_disabled)
+                if not ok:
+                    # Is this right? Cargo culting...
+                    raise wrap_windowserror(space,
+                        rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
+                resolution = resolution * time_increment
+                
+            space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+            space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+            space.setattr(w_info, space.wrap("resolution"),
+                          space.wrap(resolution))
+        return space.wrap(result)
 
 elif _MACOSX:
     c_mach_timebase_info = external('mach_timebase_info',
@@ -730,13 +783,23 @@
     timebase_info = lltype.malloc(cConfig.TIMEBASE_INFO, flavor='raw',
                                   zero=True, immortal=True)
 
-    def monotonic(space):
+    def monotonic(space, w_info=None):
         if rffi.getintfield(timebase_info, 'c_denom') == 0:
             c_mach_timebase_info(timebase_info)
         time = rffi.cast(lltype.Signed, c_mach_absolute_time())
         numer = rffi.getintfield(timebase_info, 'c_numer')
         denom = rffi.getintfield(timebase_info, 'c_denom')
         nanosecs = time * numer / denom
+        if w_info is not None:
+            space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+            space.setattr(w_info, space.wrap("implementation"),
+                          space.wrap("mach_absolute_time()"))
+            space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+            space.setattr(w_info, space.wrap("resolution"),
+                          #Do I need to convert to float indside the division?
+                          # Looking at the C, I would say yes, but nanosecs
+                          # doesn't...
+                          space.wrap((numer / denom) * 1e-9))
         secs = nanosecs / 10**9
         rest = nanosecs % 10**9
         return space.wrap(float(secs) + float(rest) * 1e-9)
@@ -744,21 +807,49 @@
 else:
     assert _POSIX
     if cConfig.CLOCK_HIGHRES is not None:
-        def monotonic(space):
+        def monotonic(space, w_info=None):
+            if w_info is not None:
+                space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("clock_gettime(CLOCK_HIGHRES)"))
+                space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+                try:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(clock_getres(space, cConfig.CLOCK_HIGHRES)))
+                except OSError:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(1e-9))
+                
             return clock_gettime(space, cConfig.CLOCK_HIGHRES)
     else:
-        def monotonic(space):
+        def monotonic(space, w_info=None):
+            if w_info is not None:
+                space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("clock_gettime(CLOCK_MONOTONIC)"))
+                space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+                try:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(clock_getres(space, cConfig.CLOCK_MONOTONIC)))
+                except OSError:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(1e-9))
+
             return clock_gettime(space, cConfig.CLOCK_MONOTONIC)
 
+if _WIN:
+    def perf_counter(space, w_info=None):
+        # What if the windows perf counter fails?
+        # Cpython falls back to monotonic and then clock
+        # Shouldn't we?
+        # TODO: Discuss on irc
 
-if _WIN:
-    def perf_counter(space):
+        # TODO: Figure out how to get at the internals of this
         return space.wrap(win_perf_counter())
 
 else:
-    def perf_counter(space):
-        return monotonic(space)
-
+    def perf_counter(space, w_info=None):
+        return monotonic(space, w_info=w_info)
 
 if _WIN:
     # untested so far
@@ -810,3 +901,54 @@
                     cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
+
+if _WIN:
+    def clock(space, w_info=None):
+        """clock() -> floating point number
+
+        Return the CPU time or real time since the start of the process or since
+        the first call to clock().  This has as much precision as the system
+        records."""
+        return space.wrap(win_perf_counter(space, w_info=w_info))
+
+else:
+    _clock = external('clock', [], clock_t)
+    def clock(space, w_info=None):
+        """clock() -> floating point number
+
+        Return the CPU time or real time since the start of the process or since
+        the first call to clock().  This has as much precision as the system
+        records."""
+        value = _clock()
+        #Is this casting correct?
+        if value == rffi.cast(clock_t, -1):
+            raise RunTimeError("the processor time used is not available "
+                               "or its value cannot be represented")
+
+        print(w_info, "INFO")
+        if w_info is not None:
+            space.setattr(w_info, space.wrap("implementation"),
+                          space.wrap("clock()"))
+            space.setattr(w_info, space.wrap("resolution"),
+                          space.wrap(1.0 / CLOCKS_PER_SEC))
+            space.setattr(w_info, space.wrap("monotonic"),
+                          space.w_True)
+            space.setattr(w_info, space.wrap("adjustable"),
+                          space.w_False)
+        return space.wrap((1.0 * value) / CLOCKS_PER_SEC)
+
+
+def get_clock_info_dict(space, name):
+    if name == "time":
+        return 5#floattime(info)
+    elif name == "monotonic":
+        return monotonic(info)
+    elif name == "clock":
+        return clock(info)
+    elif name == "perf_counter":
+        return perf_counter(info)
+    elif name == "process_time":
+        return 5#process_time(info)
+    else:
+        raise ValueError("unknown clock")
+
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -379,3 +379,21 @@
         t2 = time.process_time()
         # process_time() should not include time spent during sleep
         assert (t2 - t1) < 0.05
+
+    def test_get_clock_info_monotonic(self):
+        import time
+        clock_info = time.get_clock_info("monotonic")
+        assert clock_info.monotonic
+        assert not clock_info.adjustable
+        # Not really sure what to test about this
+        # At least this tests that the attr exists...
+        assert clock_info.resolution > 0
+
+    def test_get_clock_info_clock(self):
+        import time
+        clock_info = time.get_clock_info("clock")
+        assert clock_info.monotonic
+        assert not clock_info.adjustable
+        # Not really sure what to test about this
+        # At least this tests that the attr exists...
+        assert clock_info.resolution > 0

From pypy.commits at gmail.com  Mon Jun 20 18:16:31 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge with upstream.
Message-ID: <57686b3f.8999c20a.50ef.ffffc527@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85266:1f829b014269
Date: 2016-05-20 21:22 -0400
http://bitbucket.org/pypy/pypy/changeset/1f829b014269/

Log:	Merge with upstream.

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,4 @@
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -4674,6 +4674,7 @@
 
 
 class MiscTests(unittest.TestCase):
+    @support.cpython_only
     def test_type_lookup_mro_reference(self):
         # Issue #14199: _PyType_Lookup() has to keep a strong reference to
         # the type MRO because it may be modified during the lookup, if
diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py
--- a/lib-python/3/test/test_socket.py
+++ b/lib-python/3/test/test_socket.py
@@ -691,10 +691,11 @@
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo')
-        self.assertIn(' given)', str(cm.exception))
+        if support.check_impl_detail():
+            self.assertIn(' given)', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 0, sockname, 4)
-        self.assertIn(' given)', str(cm.exception))
+        self.assertIn(' given', str(cm.exception))
 
     def testCrucialConstants(self):
         # Testing for mission critical constants
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -93,3 +93,15 @@
 .. branch: ufunc-outer
 
 Implement ufunc.outer on numpypy
+
+.. branch: verbose-imports
+
+Support ``pypy -v``: verbose imports.  It does not log as much as
+cpython, but it should be enough to help when debugging package layout
+problems.
+
+.. branch: cpyext-macros-cast
+
+Fix some warnings when compiling CPython C extension modules
+
+.. branch: syntax_fix
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -526,6 +526,7 @@
                      unbuffered,
                      ignore_environment,
                      quiet,
+                     verbose,
                      **ignored):
     # with PyPy in top of CPython we can only have around 100
     # but we need more in the translated PyPy for the compiler package
@@ -658,6 +659,8 @@
                 inspect = True
             else:
                 # If not interactive, just read and execute stdin normally.
+                if verbose:
+                    print_banner(not no_site)
                 @hidden_applevel
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '', 'exec',
@@ -741,10 +744,10 @@
     return status
 
 def print_banner(copyright):
-    print('Python %s on %s' % (sys.version, sys.platform))
+    print('Python %s on %s' % (sys.version, sys.platform), file=sys.stderr)
     if copyright:
         print('Type "help", "copyright", "credits" or '
-              '"license" for more information.')
+              '"license" for more information.', file=sys.stderr)
 
 STDLIB_WARNING = """\
 debug: WARNING: Library path not found, using compiled-in sys.path.
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -677,9 +677,9 @@
         exc = raises(TypeError, (lambda: 0), b=3)
         assert str(exc.value) == "() got an unexpected keyword argument 'b'"
         exc = raises(TypeError, (lambda a, b: 0), 1, 2, 3, a=1)
-        assert str(exc.value) == "() takes 2 positional arguments but 3 were given"
+        assert str(exc.value) == "() got multiple values for argument 'a'"
         exc = raises(TypeError, (lambda a, b=1: 0), 1, 2, 3, a=1)
-        assert str(exc.value) == "() takes from 1 to 2 positional arguments but 3 were given"
+        assert str(exc.value) == "() got multiple values for argument 'a'"
         exc = raises(TypeError, (lambda a, **kw: 0), 1, 2, 3)
         assert str(exc.value) == "() takes 1 positional argument but 3 were given"
         exc = raises(TypeError, (lambda a, b=1, **kw: 0), 1, 2, 3)
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -51,6 +51,11 @@
                            space.newint(cache.misses.get(name, 0))])
 
 def builtinify(space, w_func):
+    """To implement at app-level modules that are, in CPython,
+    implemented in C: this decorator protects a function from being ever
+    bound like a method.  Useful because some tests do things like put
+    a "built-in" function on a class and access it via the instance.
+    """
     from pypy.interpreter.function import Function, BuiltinFunction
     func = space.interp_w(Function, w_func)
     bltn = BuiltinFunction(func)
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -12,8 +12,8 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.typedef import (
-    GetSetProperty, TypeDef, interp_attrproperty, make_weakref_descr
-)
+    GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty,
+    make_weakref_descr)
 
 
 # XXX Hack to seperate rpython and pypy
@@ -39,7 +39,7 @@
             # Linux abstract namespace
             return space.wrapbytes(path)
         else:
-            return space.wrap(path)
+            return space.wrap_fsdecoded(path)
     elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress):
         return space.newtuple([space.wrap(addr.get_pid()),
                                space.wrap(addr.get_groups())])
@@ -159,15 +159,14 @@
 
 
 class W_Socket(W_Root):
-    def __init__(self, space, sock):
+    def __init__(self, space, sock=None):
         self.space = space
-        self.sock = sock
-        register_socket(space, sock)
-
-    def descr_new(space, w_subtype, __args__):
-        sock = space.allocate_instance(W_Socket, w_subtype)
-        W_Socket.__init__(sock, space, RSocket.empty_rsocket())
-        return space.wrap(sock)
+        if sock is None:
+            self.sock = RSocket.empty_rsocket()
+        else:
+            register_socket(space, sock)
+            self.sock = sock
+            self.register_finalizer(space)
 
     @unwrap_spec(family=int, type=int, proto=int,
                  w_fileno=WrappedDefault(None))
@@ -184,12 +183,15 @@
             raise converted_error(space, e)
 
     def _finalize_(self):
-        self.clear_all_weakrefs()
-        if self.sock.fd != rsocket.INVALID_SOCKET:
+        sock = self.sock
+        if sock.fd != rsocket.INVALID_SOCKET:
             try:
                 self._dealloc_warn()
             finally:
-                self.close_w(self.space)
+                try:
+                    sock.close()
+                except SocketError:
+                    pass
 
     def get_type_w(self, space):
         return space.wrap(self.sock.type)
@@ -734,7 +736,7 @@
 shutdown(how) -- shut down traffic in one or both directions
 
  [*] not available on all platforms!""",
-    __new__ = interp2app(W_Socket.descr_new.im_func),
+    __new__ = generic_new_descr(W_Socket),
     __init__ = interp2app(W_Socket.descr_init),
     __repr__ = interp2app(W_Socket.descr_repr),
     type = GetSetProperty(W_Socket.get_type_w),
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -137,6 +137,11 @@
                   space.wrap(lib_str) if lib_str else space.w_None)
     return OperationError(w_exception_class, w_exception)
 
+def timeout_error(space, msg):
+    w_exc_class = interp_socket.get_error(space, 'timeout')
+    w_exc = space.call_function(w_exc_class, space.wrap(msg))
+    return OperationError(w_exc_class, w_exc)
+
 class SSLNpnProtocols(object):
 
     def __init__(self, ctx, protos):
@@ -334,7 +339,7 @@
 
         sockstate = checkwait(space, w_socket, True)
         if sockstate == SOCKET_HAS_TIMED_OUT:
-            raise ssl_error(space, "The write operation timed out")
+            raise timeout_error(space, "The write operation timed out")
         elif sockstate == SOCKET_HAS_BEEN_CLOSED:
             raise ssl_error(space, "Underlying socket has been closed.")
         elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
@@ -355,7 +360,7 @@
                 sockstate = SOCKET_OPERATION_OK
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The write operation timed out")
+                raise timeout_error(space, "The write operation timed out")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise ssl_error(space, "Underlying socket has been closed.")
             elif sockstate == SOCKET_IS_NONBLOCKING:
@@ -392,7 +397,7 @@
         if not count:
             sockstate = checkwait(space, w_socket, False)
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The read operation timed out")
+                raise timeout_error(space, "The read operation timed out")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
                 raise ssl_error(space,
                                 "Underlying socket too large for select().")
@@ -432,7 +437,7 @@
                     sockstate = SOCKET_OPERATION_OK
 
                 if sockstate == SOCKET_HAS_TIMED_OUT:
-                    raise ssl_error(space, "The read operation timed out")
+                    raise timeout_error(space, "The read operation timed out")
                 elif sockstate == SOCKET_IS_NONBLOCKING:
                     break
 
@@ -481,7 +486,7 @@
             else:
                 sockstate = SOCKET_OPERATION_OK
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise ssl_error(space, "The handshake operation timed out")
+                raise timeout_error(space, "The handshake operation timed out")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise ssl_error(space, "Underlying socket has been closed.")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
@@ -549,9 +554,9 @@
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
                 if ssl_err == SSL_ERROR_WANT_READ:
-                    raise ssl_error(space, "The read operation timed out")
+                    raise timeout_error(space, "The read operation timed out")
                 else:
-                    raise ssl_error(space, "The write operation timed out")
+                    raise timeout_error(space, "The write operation timed out")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
                 raise ssl_error(space,
                                 "Underlying socket too large for select().")
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -413,7 +413,16 @@
                             arg = rffi.cast(ARG, as_pyobj(space, input_arg))
                         else:
                             arg = rffi.cast(ARG, input_arg)
-                    elif is_PyObject(ARG) and is_wrapped:
+                    elif ARG == rffi.VOIDP and not is_wrapped:
+                        # unlike is_PyObject case above, we allow any kind of
+                        # argument -- just, if it's an object, we assume the
+                        # caller meant for it to become a PyObject*.
+                        if input_arg is None or isinstance(input_arg, W_Root):
+                            keepalives += (input_arg,)
+                            arg = rffi.cast(ARG, as_pyobj(space, input_arg))
+                        else:
+                            arg = rffi.cast(ARG, input_arg)
+                    elif (is_PyObject(ARG) or ARG == rffi.VOIDP) and is_wrapped:
                         # build a W_Root, possibly from a 'PyObject *'
                         if is_pyobj(input_arg):
                             arg = from_ref(space, input_arg)
@@ -725,6 +734,7 @@
 
 class WrapperGen(object):
     wrapper_second_level = None
+    A = lltype.Array(lltype.Char)
 
     def __init__(self, space, signature):
         self.space = space
@@ -737,9 +747,13 @@
         wrapper_second_level = self.wrapper_second_level
 
         name = callable.__name__
+        pname = lltype.malloc(self.A, len(name), flavor='raw', immortal=True)
+        for i in range(len(name)):
+            pname[i] = name[i]
+
         def wrapper(*args):
             # no GC here, not even any GC object
-            return wrapper_second_level(callable, name, *args)
+            return wrapper_second_level(callable, pname, *args)
 
         wrapper.__name__ = "wrapper for %r" % (callable, )
         return wrapper
@@ -747,22 +761,31 @@
 
 
 @dont_inline
+def _unpack_name(pname):
+    return ''.join([pname[i] for i in range(len(pname))])
+
+ at dont_inline
 def deadlock_error(funcname):
+    funcname = _unpack_name(funcname)
     fatalerror_notb("GIL deadlock detected when a CPython C extension "
                     "module calls '%s'" % (funcname,))
 
 @dont_inline
 def no_gil_error(funcname):
+    funcname = _unpack_name(funcname)
     fatalerror_notb("GIL not held when a CPython C extension "
                     "module calls '%s'" % (funcname,))
 
 @dont_inline
 def not_supposed_to_fail(funcname):
-    raise SystemError("The function '%s' was not supposed to fail"
-                      % (funcname,))
+    funcname = _unpack_name(funcname)
+    print "Error in cpyext, CPython compatibility layer:"
+    print "The function", funcname, "was not supposed to fail"
+    raise SystemError
 
 @dont_inline
 def unexpected_exception(funcname, e, tb):
+    funcname = _unpack_name(funcname)
     print 'Fatal error in cpyext, CPython compatibility layer, calling',funcname
     print 'Either report a bug or consider not using this particular extension'
     if not we_are_translated():
@@ -801,9 +824,8 @@
     def invalid(err):
         "NOT_RPYTHON: translation-time crash if this ends up being called"
         raise ValueError(err)
-    invalid.__name__ = 'invalid_%s' % name
 
-    def wrapper_second_level(callable, name, *args):
+    def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
         # we hope that malloc removal removes the newtuple() that is
@@ -814,7 +836,7 @@
         _gil_auto = (gil_auto_workaround and cpyext_glob_tid_ptr[0] != tid)
         if gil_acquire or _gil_auto:
             if cpyext_glob_tid_ptr[0] == tid:
-                deadlock_error(name)
+                deadlock_error(pname)
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
@@ -827,7 +849,7 @@
                 args += (pystate.PyGILState_UNLOCKED,)
         else:
             if cpyext_glob_tid_ptr[0] != tid:
-                no_gil_error(name)
+                no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
 
         rffi.stackcounter.stacks_counter += 1
@@ -844,6 +866,10 @@
                 if is_PyObject(typ) and is_wrapped:
                     assert is_pyobj(arg)
                     arg_conv = from_ref(space, rffi.cast(PyObject, arg))
+                elif typ == rffi.VOIDP and is_wrapped:
+                    # Many macros accept a void* so that one can pass a
+                    # PyObject* or a PySomeSubtype*.
+                    arg_conv = from_ref(space, rffi.cast(PyObject, arg))
                 else:
                     arg_conv = arg
                 boxed_args += (arg_conv, )
@@ -873,7 +899,7 @@
 
             if failed:
                 if error_value is CANNOT_FAIL:
-                    raise not_supposed_to_fail(name)
+                    raise not_supposed_to_fail(pname)
                 retval = error_value
 
             elif is_PyObject(restype):
@@ -893,7 +919,7 @@
                 retval = rffi.cast(restype, result)
 
         except Exception as e:
-            unexpected_exception(name, e, tb)
+            unexpected_exception(pname, e, tb)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -178,67 +178,67 @@
 
 # Accessors
 
- at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_YEAR(space, w_obj):
     """Return the year, as a positive int.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("year")))
 
- at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_MONTH(space, w_obj):
     """Return the month, as an int from 1 through 12.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("month")))
 
- at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_DAY(space, w_obj):
     """Return the day, as an int from 1 through 31.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("day")))
 
- at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("hour")))
 
- at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("minute")))
 
- at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("second")))
 
- at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("microsecond")))
 
- at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("hour")))
 
- at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("minute")))
 
- at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
     return space.int_w(space.getattr(w_obj, space.wrap("second")))
 
- at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
@@ -248,14 +248,14 @@
 # But it does not seem possible to expose a different structure
 # for types defined in a python module like lib/datetime.py.
 
- at cpython_api([PyDateTime_Delta], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_DAYS(space, w_obj):
     return space.int_w(space.getattr(w_obj, space.wrap("days")))
 
- at cpython_api([PyDateTime_Delta], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_SECONDS(space, w_obj):
     return space.int_w(space.getattr(w_obj, space.wrap("seconds")))
 
- at cpython_api([PyDateTime_Delta], rffi.INT_real, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_MICROSECONDS(space, w_obj):
     return space.int_w(space.getattr(w_obj, space.wrap("microseconds")))
diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py
--- a/pypy/module/cpyext/floatobject.py
+++ b/pypy/module/cpyext/floatobject.py
@@ -48,7 +48,7 @@
 def PyFloat_AsDouble(space, w_obj):
     return space.float_w(space.float(w_obj))
 
- at cpython_api([PyObject], lltype.Float, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], lltype.Float, error=CANNOT_FAIL)
 def PyFloat_AS_DOUBLE(space, w_float):
     """Return a C double representation of the contents of w_float, but
     without error checking."""
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM PyList_GetItem
+#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -21,7 +21,7 @@
     """
     return space.newlist([None] * len)
 
- at cpython_api([PyObject, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
 def PyList_SET_ITEM(space, w_list, index, w_item):
     """Macro form of PyList_SetItem() without error checking. This is normally
@@ -87,7 +87,7 @@
     space.call_method(space.w_list, "insert", w_list, space.wrap(index), w_item)
     return 0
 
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyList_GET_SIZE(space, w_list):
     """Macro form of PyList_Size() without error checking.
     """
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -54,7 +54,7 @@
     except OperationError:
         raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
     """Return the ith element of o, assuming that o was returned by
     PySequence_Fast(), o is not NULL, and that i is within bounds.
@@ -67,7 +67,7 @@
                 "PySequence_Fast_GET_ITEM called but object is not a list or "
                 "sequence")
 
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PySequence_Fast_GET_SIZE(space, w_obj):
     """Returns the length of o, assuming that o was returned by
     PySequence_Fast() and that o is not NULL.  The size can also be
@@ -82,7 +82,7 @@
                 "PySequence_Fast_GET_SIZE called but object is not a list or "
                 "sequence")
 
- at cpython_api([PyObject], PyObjectP)
+ at cpython_api([rffi.VOIDP], PyObjectP)
 def PySequence_Fast_ITEMS(space, w_obj):
     """Return the underlying array of PyObject pointers.  Assumes that o was returned
     by PySequence_Fast() and o is not NULL.
@@ -119,7 +119,7 @@
     space.delslice(w_obj, space.wrap(start), space.wrap(end))
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject)
 def PySequence_ITEM(space, w_obj, i):
     """Return the ith element of o or NULL on failure. Macro form of
     PySequence_GetItem() but without checking that
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -74,7 +74,7 @@
     space.call_method(space.w_set, 'clear', w_set)
     return 0
 
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PySet_GET_SIZE(space, w_s):
     """Macro form of PySet_Size() without error checking."""
     return space.int_w(space.len(w_s))
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -117,3 +117,108 @@
                                       datetime.timedelta,
                                       datetime.tzinfo)
         module.clear_types()
+
+    def test_macros(self):
+        module = self.import_extension('foo', [
+            ("test_date_macros", "METH_NOARGS",
+             """
+                 PyObject* obj;
+                 PyDateTime_Date* d;
+                 PyDateTime_IMPORT;
+                 if (!PyDateTimeAPI) {
+                     PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI");
+                     return NULL;
+                 }
+                 obj = PyDate_FromDate(2000, 6, 6);
+                 d = (PyDateTime_Date*)obj;
+
+                 PyDateTime_GET_YEAR(obj);
+                 PyDateTime_GET_YEAR(d);
+
+                 PyDateTime_GET_MONTH(obj);
+                 PyDateTime_GET_MONTH(d);
+
+                 PyDateTime_GET_DAY(obj);
+                 PyDateTime_GET_DAY(d);
+
+                 return obj;
+             """),
+            ("test_datetime_macros", "METH_NOARGS",
+             """
+                 PyDateTime_IMPORT;
+                 if (!PyDateTimeAPI) {
+                     PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI");
+                     return NULL;
+                 }
+                 PyObject* obj = PyDateTime_FromDateAndTime(2000, 6, 6, 6, 6, 6, 6);
+                 PyDateTime_DateTime* dt = (PyDateTime_DateTime*)obj;
+
+                 PyDateTime_GET_YEAR(obj);
+                 PyDateTime_GET_YEAR(dt);
+
+                 PyDateTime_GET_MONTH(obj);
+                 PyDateTime_GET_MONTH(dt);
+
+                 PyDateTime_GET_DAY(obj);
+                 PyDateTime_GET_DAY(dt);
+
+                 PyDateTime_DATE_GET_HOUR(obj);
+                 PyDateTime_DATE_GET_HOUR(dt);
+
+                 PyDateTime_DATE_GET_MINUTE(obj);
+                 PyDateTime_DATE_GET_MINUTE(dt);
+
+                 PyDateTime_DATE_GET_SECOND(obj);
+                 PyDateTime_DATE_GET_SECOND(dt);
+
+                 PyDateTime_DATE_GET_MICROSECOND(obj);
+                 PyDateTime_DATE_GET_MICROSECOND(dt);
+
+                 return obj;
+             """),
+            ("test_time_macros", "METH_NOARGS",
+             """
+                 PyDateTime_IMPORT;
+                 if (!PyDateTimeAPI) {
+                     PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI");
+                     return NULL;
+                 }
+                 PyObject* obj = PyTime_FromTime(6, 6, 6, 6);
+                 PyDateTime_Time* t = (PyDateTime_Time*)obj;
+
+                 PyDateTime_TIME_GET_HOUR(obj);
+                 PyDateTime_TIME_GET_HOUR(t);
+
+                 PyDateTime_TIME_GET_MINUTE(obj);
+                 PyDateTime_TIME_GET_MINUTE(t);
+
+                 PyDateTime_TIME_GET_SECOND(obj);
+                 PyDateTime_TIME_GET_SECOND(t);
+
+                 PyDateTime_TIME_GET_MICROSECOND(obj);
+                 PyDateTime_TIME_GET_MICROSECOND(t);
+
+                 return obj;
+             """),
+            ("test_delta_macros", "METH_NOARGS",
+             """
+                 PyDateTime_IMPORT;
+                 if (!PyDateTimeAPI) {
+                     PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI");
+                     return NULL;
+                 }
+                 PyObject* obj = PyDelta_FromDSU(6, 6, 6);
+                 PyDateTime_Delta* delta = (PyDateTime_Delta*)obj;
+
+                 PyDateTime_DELTA_GET_DAYS(obj);
+                 PyDateTime_DELTA_GET_DAYS(delta);
+
+                 PyDateTime_DELTA_GET_SECONDS(obj);
+                 PyDateTime_DELTA_GET_SECONDS(delta);
+
+                 PyDateTime_DELTA_GET_MICROSECONDS(obj);
+                 PyDateTime_DELTA_GET_MICROSECONDS(delta);
+
+                 return obj;
+             """),
+            ])
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -77,3 +77,19 @@
         neginf = module.return_neginf()
         assert neginf < 0
         assert math.isinf(neginf)
+
+    def test_macro_accepts_wrong_pointer_type(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("test_macros", "METH_NOARGS",
+             """
+             PyObject* o = PyFloat_FromDouble(1.0);
+             // no PyFloatObject
+             char* dumb_pointer = (char*)o;
+
+             PyFloat_AS_DOUBLE(o);
+             PyFloat_AS_DOUBLE(dumb_pointer);
+
+             Py_RETURN_NONE;"""),
+            ])
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -137,6 +137,33 @@
         module.setlistitem(l,0)
         assert l == [None, 2, 3]
 
+    def test_list_macros(self):
+        """The PyList_* macros cast, and calls expecting that build."""
+        module = self.import_extension('foo', [
+            ("test_macro_invocations", "METH_NOARGS",
+             """
+             PyObject* o = PyList_New(2);
+             PyListObject* l = (PyListObject*)o;
+
+
+             Py_INCREF(o);
+             PyList_SET_ITEM(o, 0, o);
+             Py_INCREF(o);
+             PyList_SET_ITEM(l, 1, o);
+
+             PyList_GET_ITEM(o, 0);
+             PyList_GET_ITEM(l, 1);
+
+             PyList_GET_SIZE(o);
+             PyList_GET_SIZE(l);
+
+             return o;
+             """
+            )
+        ])
+        x = module.test_macro_invocations()
+        assert x[0] is x[1] is x
+
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
              ("test_get_item", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -155,6 +155,29 @@
         result = api.PySequence_Index(w_gen, w_tofind)
         assert result == 4
 
+class AppTestSetObject(AppTestCpythonExtensionBase):
+    def test_sequence_macro_cast(self):
+        module = self.import_extension('foo', [
+            ("test_macro_cast", "METH_NOARGS",
+             """
+             PyObject *o = PyList_New(0);
+             PyListObject* l;
+             PyList_Append(o, o);
+             l = (PyListObject*)o;
+
+             PySequence_Fast_GET_ITEM(o, 0);
+             PySequence_Fast_GET_ITEM(l, 0);
+
+             PySequence_Fast_GET_SIZE(o);
+             PySequence_Fast_GET_SIZE(l);
+
+             PySequence_ITEM(o, 0);
+             PySequence_ITEM(l, 0);
+
+             return o;
+             """
+            )
+        ])
 class TestCPyListStrategy(BaseApiTest):
     def test_getitem_setitem(self, space, api):
         w_l = space.wrap([1, 2, 3, 4])
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -2,6 +2,7 @@
 
 from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 
 
@@ -45,3 +46,20 @@
         w_frozenset = space.newfrozenset([space.wrap(i) for i in [1, 2, 3, 4]])
         assert api.PyAnySet_CheckExact(w_set)
         assert api.PyAnySet_CheckExact(w_frozenset)
+
+class AppTestSetObject(AppTestCpythonExtensionBase):
+    def test_set_macro_cast(self):
+        module = self.import_extension('foo', [
+            ("test_macro_cast", "METH_NOARGS",
+             """
+             PyObject* o = PySet_New(NULL);
+             // no PySetObject
+             char* dumb_pointer = (char*) o;
+
+             PySet_GET_SIZE(o);
+             PySet_GET_SIZE(dumb_pointer);
+
+             return o;
+             """
+            )
+        ])
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -160,6 +160,26 @@
         assert module.compare("abc", b"") == 1
 
 
+    def test_unicode_macros(self):
+        """The PyUnicode_* macros cast, and calls expecting that build."""
+        module = self.import_extension('foo', [
+             ("test_macro_invocations", "METH_NOARGS",
+             """
+                PyObject* o = PyUnicode_FromString("");
+                PyUnicodeObject* u = (PyUnicodeObject*)o;
+
+                PyUnicode_GET_SIZE(u);
+                PyUnicode_GET_SIZE(o);
+
+                PyUnicode_GET_DATA_SIZE(u);
+                PyUnicode_GET_DATA_SIZE(o);
+
+                PyUnicode_AS_UNICODE(o);
+                PyUnicode_AS_UNICODE(u);
+                return o;
+             """)])
+        assert module.test_macro_invocations() == u''
+
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space, api):
         assert api.PyUnicode_GET_SIZE(space.wrap(u'späm')) == 4
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test/test_weakref.py
--- a/pypy/module/cpyext/test/test_weakref.py
+++ b/pypy/module/cpyext/test/test_weakref.py
@@ -7,7 +7,6 @@
         w_ref = api.PyWeakref_NewRef(w_obj, space.w_None)
         assert w_ref is not None
         assert space.is_w(api.PyWeakref_GetObject(w_ref), w_obj)
-        assert space.is_w(api.PyWeakref_GET_OBJECT(w_ref), w_obj)
         assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj)
 
         w_obj = space.newtuple([])
@@ -34,3 +33,26 @@
         del w_obj
         import gc; gc.collect()
         assert space.is_w(api.PyWeakref_LockObject(w_ref), space.w_None)
+
+
+class AppTestWeakReference(AppTestCpythonExtensionBase):
+
+    def test_weakref_macro(self):
+        module = self.import_extension('foo', [
+            ("test_macro_cast", "METH_NOARGS",
+             """
+             // PyExc_Warning is some weak-reffable PyObject*.
+             char* dumb_pointer;
+             PyObject* weakref_obj = PyWeakref_NewRef(PyExc_Warning, NULL);
+             if (!weakref_obj) return weakref_obj;
+             // No public PyWeakReference type.
+             dumb_pointer = (char*) weakref_obj;
+
+             PyWeakref_GET_OBJECT(weakref_obj);
+             PyWeakref_GET_OBJECT(dumb_pointer);
+
+             return weakref_obj;
+             """
+            )
+        ])
+        module.test_macro_cast()
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -18,8 +18,9 @@
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
     PyObjectFields, Py_TPFLAGS_BASETYPE, Py_buffer)
-from pypy.module.cpyext.methodobject import (
-    PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef)
+from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
+    PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
+    W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
@@ -125,6 +126,14 @@
 cpython_struct("PyGetSetDescrObject", PyGetSetDescrObjectFields,
                PyGetSetDescrObjectStruct, level=2)
 
+PyMethodDescrObjectStruct = lltype.ForwardReference()
+PyMethodDescrObject = lltype.Ptr(PyMethodDescrObjectStruct)
+PyMethodDescrObjectFields = PyDescrObjectFields + (
+    ("d_method", lltype.Ptr(PyMethodDef)),
+    )
+cpython_struct("PyMethodDescrObject", PyMethodDescrObjectFields,
+               PyMethodDescrObjectStruct, level=2)
+
 @bootstrap_function
 def init_memberdescrobject(space):
     make_typedescr(W_MemberDescr.typedef,
@@ -136,6 +145,16 @@
                    basestruct=PyGetSetDescrObject.TO,
                    attach=getsetdescr_attach,
                    )
+    make_typedescr(W_PyCClassMethodObject.typedef,
+                   basestruct=PyMethodDescrObject.TO,
+                   attach=methoddescr_attach,
+                   realize=classmethoddescr_realize,
+                   )
+    make_typedescr(W_PyCMethodObject.typedef,
+                   basestruct=PyMethodDescrObject.TO,
+                   attach=methoddescr_attach,
+                   realize=methoddescr_realize,
+                   )
 
 def memberdescr_attach(space, py_obj, w_obj):
     """
@@ -166,6 +185,30 @@
     assert isinstance(w_obj, W_GetSetPropertyEx)
     py_getsetdescr.c_d_getset = w_obj.getset
 
+def methoddescr_attach(space, py_obj, w_obj):
+    py_methoddescr = rffi.cast(PyMethodDescrObject, py_obj)
+    # XXX assign to d_dname, d_type?
+    assert isinstance(w_obj, W_PyCFunctionObject)
+    py_methoddescr.c_d_method = w_obj.ml
+
+def classmethoddescr_realize(space, obj):
+    # XXX NOT TESTED When is this ever called? 
+    method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_PyCClassMethodObject, w_type)
+    w_obj.__init__(space, method, w_type)
+    track_reference(space, obj, w_obj)
+    return w_obj
+
+def methoddescr_realize(space, obj):
+    # XXX NOT TESTED When is this ever called? 
+    method = rffi.cast(lltype.Ptr(PyMethodDef), obj)
+    w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+    w_obj = space.allocate_instance(W_PyCMethodObject, w_type)
+    w_obj.__init__(space, method, w_type)
+    track_reference(space, obj, w_obj)
+    return w_obj
+
 def convert_getset_defs(space, dict_w, getsets, w_type):
     getsets = rffi.cast(rffi.CArrayPtr(PyGetSetDef), getsets)
     if getsets:
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -183,19 +183,19 @@
     """Get the maximum ordinal for a Unicode character."""
     return runicode.UNICHR(runicode.MAXUNICODE)
 
- at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=CANNOT_FAIL)
 def PyUnicode_AS_DATA(space, ref):
     """Return a pointer to the internal buffer of the object. o has to be a
     PyUnicodeObject (not checked)."""
     return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref))
 
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_DATA_SIZE(space, w_obj):
     """Return the size of the object's internal buffer in bytes.  o has to be a
     PyUnicodeObject (not checked)."""
     return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, w_obj)
 
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+ at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
 def PyUnicode_GET_SIZE(space, w_obj):
     """Return the size of the object.  o has to be a PyUnicodeObject (not
     checked)."""
@@ -222,7 +222,7 @@
     ref_unicode = rffi.cast(PyUnicodeObject, ref)
     if not ref_unicode.c_buffer:
         # Copy unicode buffer
-        w_unicode = from_ref(space, ref)
+        w_unicode = from_ref(space, rffi.cast(PyObject, ref))
         u = space.unicode_w(w_unicode)
         ref_unicode.c_buffer = rffi.unicode2wcharp(u)
     return ref_unicode.c_buffer
@@ -235,7 +235,7 @@
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
     if not space.is_true(space.issubtype(w_type, space.w_unicode)):
         raise oefmt(space.w_TypeError, "expected unicode object")
-    return PyUnicode_AS_UNICODE(space, ref)
+    return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
 
 @cpython_api([PyObject], rffi.CCHARP)
 def _PyUnicode_AsString(space, ref):
@@ -267,8 +267,8 @@
     string may or may not be 0-terminated.  It is the responsibility of the caller
     to make sure that the wchar_t string is 0-terminated in case this is
     required by the application."""
-    c_buffer = PyUnicode_AS_UNICODE(space, ref)
     ref = rffi.cast(PyUnicodeObject, ref)
+    c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
     c_length = ref.c_length
 
     # If possible, try to copy the 0-termination as well
diff --git a/pypy/module/cpyext/weakrefobject.py b/pypy/module/cpyext/weakrefobject.py
--- a/pypy/module/cpyext/weakrefobject.py
+++ b/pypy/module/cpyext/weakrefobject.py
@@ -1,6 +1,7 @@
 from pypy.module.cpyext.api import cpython_api
 from pypy.module.cpyext.pyobject import PyObject
 from pypy.module._weakref.interp__weakref import W_Weakref, proxy
+from rpython.rtyper.lltypesystem import rffi
 
 @cpython_api([PyObject, PyObject], PyObject)
 def PyWeakref_NewRef(space, w_obj, w_callback):
@@ -37,7 +38,7 @@
     """
     return space.call_function(w_ref)     # borrowed ref
 
- at cpython_api([PyObject], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP], PyObject, result_borrowed=True)
 def PyWeakref_GET_OBJECT(space, w_ref):
     """Similar to PyWeakref_GetObject(), but implemented as a macro that does no
     error checking.
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -41,6 +41,14 @@
 
     return '.' + soabi + SO
 
+def log_pyverbose(space, level, message):
+    if space.sys.w_initialdict is None:
+        return # sys module not initialised, avoid recursion
+    verbose = space.sys.get_flag('verbose')
+    if verbose >= level:
+        w_stderr = space.sys.get('stderr')
+        space.call_method(w_stderr, "write", space.wrap(message))
+
 def has_so_extension(space):
     return (space.config.objspace.usemodules.cpyext or
             space.config.objspace.usemodules._cffi_backend)
@@ -354,6 +362,9 @@
     Load a module from a compiled file, execute it, and return its
     module object.
     """
+    log_pyverbose(space, 1, "import %s # compiled from %s\n" %
+                  (space.str_w(w_modulename), cpathname))
+
     if magic != get_pyc_magic(space):
         raise oefmt(space.w_ImportError, "Bad magic number in %s", cpathname)
     #print "loading pyc file:", cpathname
diff --git a/pypy/module/imp/test/support.py b/pypy/module/imp/test/support.py
--- a/pypy/module/imp/test/support.py
+++ b/pypy/module/imp/test/support.py
@@ -4,14 +4,57 @@
 
     def setup_class(cls):
         space = cls.space
-        testfn = u'test_tmp'
-        testfn_unencodable = None
+        cls.w_testfn_unencodable = space.wrap(get_unencodable())
+        cls.w_special_char = space.wrap(get_special_char())
 
-        if sys.platform == 'win32':
-            testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
-        elif sys.platform != 'darwin':
-            try:
-                '\xff'.decode(sys.getfilesystemencoding())
-            except UnicodeDecodeError:
-                testfn_unencodable = testfn + u'-\udcff'
-        cls.w_testfn_unencodable = space.wrap(testfn_unencodable)
+def get_unencodable():
+    """Copy of the stdlib's support.TESTFN_UNENCODABLE:
+
+    A filename (py3k str type) that should *not* be able to be encoded
+    by the filesystem encoding (in strict mode). It can be None if we
+    cannot generate such filename.
+    """
+    testfn_unencodable = None
+    testfn = u'test_tmp'
+
+    if sys.platform == 'win32':
+        testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
+    elif sys.platform != 'darwin':
+        try:
+            '\xff'.decode(sys.getfilesystemencoding())
+        except UnicodeDecodeError:
+            testfn_unencodable = testfn + u'-\udcff'
+    return testfn_unencodable
+
+def get_special_char():
+    """Copy of the stdlib's test_imp.test_issue5604 special_char:
+
+    A non-ascii filename (py3k str type) that *should* be able to be
+    encoded by the filesystem encoding (in strict mode). It can be None
+    if we cannot generate such filename.
+    """
+    fsenc = sys.getfilesystemencoding()
+    # covers utf-8 and Windows ANSI code pages one non-space symbol from
+    # every page (http://en.wikipedia.org/wiki/Code_page)
+    known_locales = {
+        'utf-8' : b'\xc3\xa4',
+        'cp1250' : b'\x8C',
+        'cp1251' : b'\xc0',
+        'cp1252' : b'\xc0',
+        'cp1253' : b'\xc1',
+        'cp1254' : b'\xc0',
+        'cp1255' : b'\xe0',
+        'cp1256' : b'\xe0',
+        'cp1257' : b'\xc0',
+        'cp1258' : b'\xc0',
+        }
+
+    if sys.platform == 'darwin':
+        # Mac OS X uses the Normal Form D decomposition
+        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
+        special_char = b'a\xcc\x88'
+    else:
+        special_char = known_locales.get(fsenc)
+
+    if special_char:
+        return special_char.decode(fsenc)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -46,15 +46,13 @@
     if pkgname:
         p = p.join(*pkgname.split('.'))
     p.ensure(dir=1)
-    f = p.join("__init__.py").open('w')
-    print >> f, "# package"
-    f.close()
+    with p.join("__init__.py").open('w') as f:
+        print >> f, "# package"
     for filename, content in entries.items():
         filename += '.py'
-        f = p.join(filename).open('w')
-        print >> f, '#', filename
-        print >> f, content
-        f.close()
+        with p.join(filename).open('w') as f:
+            print >> f, '#', filename
+            print >> f, content
     return p
 
 def setup_directory_structure(cls):
@@ -123,6 +121,9 @@
         'a=5\nb=6\rc="""hello\r\nworld"""\r', mode='wb')
     p.join('mod.py').write(
         'a=15\nb=16\rc="""foo\r\nbar"""\r', mode='wb')
+    setuppkg("verbose1pkg", verbosemod='a = 1729')
+    setuppkg("verbose2pkg", verbosemod='a = 1729')
+    setuppkg("verbose0pkg", verbosemod='a = 1729')
     setuppkg("test_bytecode",
              a = '',
              b = '',
@@ -132,34 +133,11 @@
              line2 = "# encoding: iso-8859-1\n",
              bad = "# encoding: uft-8\n")
 
-    fsenc = sys.getfilesystemencoding()
-    # covers utf-8 and Windows ANSI code pages one non-space symbol from
-    # every page (http://en.wikipedia.org/wiki/Code_page)
-    known_locales = {
-        'utf-8' : b'\xc3\xa4',
-        'cp1250' : b'\x8C',
-        'cp1251' : b'\xc0',
-        'cp1252' : b'\xc0',
-        'cp1253' : b'\xc1',
-        'cp1254' : b'\xc0',
-        'cp1255' : b'\xe0',
-        'cp1256' : b'\xe0',
-        'cp1257' : b'\xc0',
-        'cp1258' : b'\xc0',
-        }
-
-    if sys.platform == 'darwin':
-        # Mac OS X uses the Normal Form D decomposition
-        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
-        special_char = b'a\xcc\x88'
-    else:
-        special_char = known_locales.get(fsenc)
-
-    if special_char:
+    w_special_char = getattr(cls, 'w_special_char', None)
+    if not space.is_none(w_special_char):
+        special_char = space.unicode_w(w_special_char).encode(
+            sys.getfilesystemencoding())
         p.join(special_char + '.py').write('pass')
-        cls.w_special_char = space.wrap(special_char.decode(fsenc))
-    else:
-        cls.w_special_char = space.w_None
 
     # create a .pyw file
     p = setuppkg("windows", x = "x = 78")
@@ -588,9 +566,8 @@
         import test_reload
         import time, imp
         time.sleep(1)
-        f = open(test_reload.__file__, "w")
-        f.write("a = 10 // 0\n")
-        f.close()
+        with open(test_reload.__file__, "w") as f:
+            f.write("a = 10 // 0\n")
 
         # A failing reload should leave the previous module in sys.modules
         raises(ZeroDivisionError, imp.reload, test_reload)
@@ -733,7 +710,8 @@
         import pkg
         import os
         pathname = os.path.join(os.path.dirname(pkg.__file__), 'a.py')
-        module = imp.load_module('a', open(pathname),
+        with open(pathname) as fid:
+            module = imp.load_module('a', fid,
                                  'invalid_path_name', ('.py', 'r', imp.PY_SOURCE))
         assert module.__name__ == 'a'
         assert module.__file__ == 'invalid_path_name'
@@ -768,6 +746,68 @@
             else:
                 raise AssertionError("should have failed")
 
+    def test_verbose_flag_1(self):
+        output = []
+        class StdErr(object):
+            def write(self, line):
+                output.append(line)
+
+        import sys, imp
+        old_flags = sys.flags
+
+        class Flags(object):
+            verbose = 1
+            def __getattr__(self, name):
+                return getattr(old_flags, name)
+
+        sys.flags = Flags()
+        sys.stderr = StdErr()
+        try:
+            import verbose1pkg.verbosemod
+        finally:
+            imp.reload(sys)
+        assert 'import verbose1pkg # ' in output[-2]
+        assert 'import verbose1pkg.verbosemod # ' in output[-1]
+
+    def test_verbose_flag_2(self):
+        output = []
+        class StdErr(object):
+            def write(self, line):
+                output.append(line)
+
+        import sys, imp
+        old_flags = sys.flags
+
+        class Flags(object):
+            verbose = 2
+            def __getattr__(self, name):
+                return getattr(old_flags, name)
+
+        sys.flags = Flags()
+        sys.stderr = StdErr()
+        try:
+            import verbose2pkg.verbosemod
+        finally:
+            imp.reload(sys)
+        assert any('import verbose2pkg # ' in line
+                   for line in output[:-2])
+        assert output[-2].startswith('# trying')
+        assert 'import verbose2pkg.verbosemod # ' in output[-1]
+
+    def test_verbose_flag_0(self):
+        output = []
+        class StdErr(object):
+            def write(self, line):
+                output.append(line)
+
+        import sys, imp
+        sys.stderr = StdErr()
+        try:
+            import verbose0pkg.verbosemod
+        finally:
+            imp.reload(sys)
+        assert not output
+
     def test_source_encoding(self):
         import imp
         import encoded
@@ -781,9 +821,9 @@
         raises(SyntaxError, imp.find_module, 'bad', encoded.__path__)
 
     def test_find_module_fsdecode(self):
-        import sys
         name = self.special_char
         if not name:
+            import sys
             skip("can't run this test with %s as filesystem encoding"
                  % sys.getfilesystemencoding())
         import imp
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -349,14 +349,23 @@
         assert sys.path_hooks.count(zipimport.zipimporter) == 1
 
     def w__make_unicode_filename(self):
+        if not self.testfn_unencodable:
+            import sys
+            skip("can't run this test with %s as filesystem encoding"
+                 % sys.getfilesystemencoding())
         import os
         head, tail = os.path.split(self.zipfile)
-        self.zipfile = head + os.path.sep + tail[:4] + '_ä' + tail[4:]
+        self.zipfile = (head + os.path.sep + tail[:4] +
+                        self.testfn_unencodable + tail[4:])
 
     def test_unicode_filename_notfound(self):
+        if not self.special_char:
+            import sys
+            skip("can't run this test with %s as filesystem encoding"
+                 % sys.getfilesystemencoding())
         import zipimport
         raises(zipimport.ZipImportError,
-               zipimport.zipimporter, 'caf\xe9')
+               zipimport.zipimporter, self.special_char)
 
     def test_unicode_filename_invalid_zippath(self):
         import zipimport
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1071,6 +1071,16 @@
         class D(A, B):     # "best base" is A
             __slots__ = ("__weakref__",)
 
+    def test_slot_shadows_class_variable(self):
+        try:
+            class X:
+                __slots__ = ["foo"]
+                foo = None
+        except ValueError as e:
+            assert str(e) == "'foo' in __slots__ conflicts with class variable"
+        else:
+            assert False, "ValueError expected"
+
     def test_metaclass_calc(self):
         """
         # issue1294232: correct metaclass calculation
@@ -1318,15 +1328,6 @@
 
         assert b == 1
 
-    def test_slots_with_method_in_class(self):
-        # this works in cpython...
-        class A(object):
-            __slots__ = ["f"]
-            def f(self, x):
-                return x + 1
-        a = A()
-        assert a.f(1) == 2
-
     def test_eq_returns_notimplemented(self):
         assert type.__eq__(int, 42) is NotImplemented
         assert type.__ne__(dict, 42) is NotImplemented
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1041,7 +1041,8 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, slot_name,
+                index_next_extra_slot = create_slot(w_self, w_slot_name,
+                                                    slot_name,
                                                     index_next_extra_slot)
     wantdict = wantdict or hasoldstylebase
     if wantdict:
@@ -1057,13 +1058,17 @@
         return Layout(base_layout.typedef, index_next_extra_slot,
                       base_layout=base_layout)
 
-def create_slot(w_self, slot_name, index_next_extra_slot):
+def create_slot(w_self, w_slot_name, slot_name, index_next_extra_slot):
     space = w_self.space
     if not valid_slot_name(slot_name):
         raise oefmt(space.w_TypeError, "__slots__ must be identifiers")
     # create member
     slot_name = mangle(slot_name, w_self.name)
-    if slot_name not in w_self.dict_w:
+    if slot_name in w_self.dict_w:
+        raise oefmt(space.w_ValueError,
+                    "%R in __slots__ conflicts with class variable",
+                    w_slot_name)
+    else:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,26 +1,33 @@
 # Edit these appropriately before running this script
 maj=5
 min=1
-rev=1
+rev=2
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-$maj.$min.$rev  # ==OR== release-$maj.$min
 
+echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
+echo checking hg log -r $tagname
 hg log -r $tagname || exit 1
 
 # This script will download latest builds from the buildmaster, rename the top
 # level directory, and repackage ready to be uploaded to bitbucket. It will also
 # download source, assuming a tag for the release already exists, and repackage them.
 # The script should be run in an empty directory, i.e. /tmp/release_xxx
-
 for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64 s390x
   do
+    echo downloading package for $plat
     wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.tar.bz2
     tar -xf pypy-c-jit-latest-$plat.tar.bz2
     rm pypy-c-jit-latest-$plat.tar.bz2
-    mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat
-    tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-$plat.tar.bz2 pypy-$maj.$min.$rev-$plat
-    rm -rf pypy-$maj.$min.$rev-$plat
+    plat_final=$plat
+    if [ $plat = linux ]; then
+        plat_final=linux32
+    fi
+    mv pypy-c-jit-*-$plat pypy-$maj.$min.$rev-$plat_final
+    echo packaging $plat_final
+    tar --owner=root --group=root --numeric-owner -cvjf pypy-$maj.$min.$rev-$plat_final.tar.bz2 pypy-$maj.$min.$rev-$plat_final
+    rm -rf pypy-$maj.$min.$rev-$plat_final
   done
 
 plat=win32
diff --git a/pypy/tool/test/test_tab.py b/pypy/tool/test/test_tab.py
--- a/pypy/tool/test/test_tab.py
+++ b/pypy/tool/test/test_tab.py
@@ -7,7 +7,11 @@
 
 ROOT = os.path.abspath(os.path.join(pypydir, '..'))
 RPYTHONDIR = os.path.join(ROOT, "rpython")
-EXCLUDE = {'/virt_test/lib/python2.7/site-packages/setuptools'}
+
+EXCLUDE = {'/virt_test'}
+# ^^^ don't look inside this: it is created by virtualenv on buildslaves.
+# It contains third-party installations that may include tabs in their
+# .py files.
 
 
 def test_no_tabs():
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -27,13 +27,13 @@
     """NOT_RPYTHON: set up rawrefcount with the GC.  This is only used
     for tests; it should not be called at all during translation.
     """
-    global _p_list, _o_list, _adr2pypy, _pypy2ob, _ob_set
+    global _p_list, _o_list, _adr2pypy, _pypy2ob, _pypy2ob_rev
     global _d_list, _dealloc_trigger_callback
     _p_list = []
     _o_list = []
     _adr2pypy = [None]
     _pypy2ob = {}
-    _ob_set = set()
+    _pypy2ob_rev = {}
     _d_list = []
     _dealloc_trigger_callback = dealloc_trigger_callback
 
@@ -41,23 +41,22 @@
     "NOT_RPYTHON: a link where the PyPy object contains some or all the data"
     #print 'create_link_pypy\n\t%s\n\t%s' % (p, ob)
     assert p not in _pypy2ob
-    assert ob._obj not in _ob_set
+    assert ob._obj not in _pypy2ob_rev
     assert not ob.c_ob_pypy_link
     ob.c_ob_pypy_link = _build_pypy_link(p)
     _pypy2ob[p] = ob
+    _pypy2ob_rev[ob._obj] = p
     _p_list.append(ob)
-    _ob_set.add(ob._obj)
 
 def create_link_pyobj(p, ob):
     """NOT_RPYTHON: a link where the PyObject contains all the data.
        from_obj() will not work on this 'p'."""
     #print 'create_link_pyobj\n\t%s\n\t%s' % (p, ob)
     assert p not in _pypy2ob
-    assert ob._obj not in _ob_set
+    assert ob._obj not in _pypy2ob_rev
     assert not ob.c_ob_pypy_link
     ob.c_ob_pypy_link = _build_pypy_link(p)
     _o_list.append(ob)
-    _ob_set.add(ob._obj)
 
 def from_obj(OB_PTR_TYPE, p):
     "NOT_RPYTHON"
@@ -65,6 +64,7 @@
     if ob is None:
         return lltype.nullptr(OB_PTR_TYPE.TO)
     assert lltype.typeOf(ob) == OB_PTR_TYPE
+    assert _pypy2ob_rev[ob._obj] is p
     return ob
 
 def to_obj(Class, ob):
@@ -111,8 +111,10 @@
             new_p_list.append(ob)
         else:
             p = detach(ob, wr_p_list)
-            del _pypy2ob[p]
-            del p
+            ob_test = _pypy2ob.pop(p)
+            p_test = _pypy2ob_rev.pop(ob_test._obj)
+            assert p_test is p
+            del p, p_test
         ob = None
     _p_list = Ellipsis
 
@@ -156,6 +158,10 @@
         p = attach(ob, wr, _p_list)
         if p is not None:
             _pypy2ob[p] = ob
+    _pypy2ob_rev.clear()       # rebuild this dict from scratch
+    for p, ob in _pypy2ob.items():
+        assert ob._obj not in _pypy2ob_rev
+        _pypy2ob_rev[ob._obj] = p
     _o_list = []
     for ob, wr in wr_o_list:
         attach(ob, wr, _o_list)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1045,15 +1045,23 @@
         win32traits = make_win32_traits(traits)
         path1 = traits.as_str0(path1)
         path2 = traits.as_str0(path2)
-        if not win32traits.MoveFile(path1, path2):
+        if not win32traits.MoveFileEx(path1, path2, 0):
             raise rwin32.lastSavedWindowsError()
 
 @specialize.argtype(0, 1)
 def replace(path1, path2):
-    if os.name == 'nt':
-        raise NotImplementedError(
-            'On windows, os.replace() should overwrite the destination')
-    return rename(path1, path2)
+    if _WIN32:
+        traits = _preferred_traits(path1)
+        win32traits = make_win32_traits(traits)
+        path1 = traits.as_str0(path1)
+        path2 = traits.as_str0(path2)
+        ret = win32traits.MoveFileEx(path1, path2,
+                     win32traits.MOVEFILE_REPLACE_EXISTING)
+        if not ret:
+            raise rwin32.lastSavedWindowsError()
+    else:
+        ret = rename(path1, path2)
+    return ret
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -45,6 +45,8 @@
             'INVALID_FILE_ATTRIBUTES')
         ERROR_SHARING_VIOLATION = platform.ConstantInteger(
             'ERROR_SHARING_VIOLATION')
+        MOVEFILE_REPLACE_EXISTING = platform.ConstantInteger(
+            'MOVEFILE_REPLACE_EXISTING')
         _S_IFDIR = platform.ConstantInteger('_S_IFDIR')
         _S_IFREG = platform.ConstantInteger('_S_IFREG')
         _S_IFCHR = platform.ConstantInteger('_S_IFCHR')
@@ -103,7 +105,7 @@
                        FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
                        VOLUME_NAME_DOS VOLUME_NAME_NT
                        ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
-                       ERROR_SHARING_VIOLATION
+                       ERROR_SHARING_VIOLATION MOVEFILE_REPLACE_EXISTING
                     '''.split():
             locals()[name] = config[name]
         LPWIN32_FIND_DATA    = lltype.Ptr(WIN32_FIND_DATA)
@@ -199,9 +201,9 @@
             rwin32.BOOL,
             save_err=rffi.RFFI_SAVE_LASTERROR)
 
-        MoveFile = external(
-            'MoveFile' + suffix,
-            [traits.CCHARP, traits.CCHARP],
+        MoveFileEx = external(
+            'MoveFileEx' + suffix,
+            [traits.CCHARP, traits.CCHARP, rwin32.DWORD],
             rwin32.BOOL,
             save_err=rffi.RFFI_SAVE_LASTERROR)
 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -334,6 +334,11 @@
             self.path  = UnicodeWithEncoding(self.ufilename)
             self.path2 = UnicodeWithEncoding(self.ufilename + ".new")
 
+    def _teardown_method(self, method):
+        for path in [self.ufilename + ".new", self.ufilename]:
+            if os.path.exists(path):
+                os.unlink(path)
+
     def test_open(self):
         def f():
             try:
@@ -390,6 +395,14 @@
         assert not os.path.exists(self.ufilename)
         assert os.path.exists(self.ufilename + '.new')
 
+    def test_replace(self):
+        def f():
+            return rposix.replace(self.path, self.path2)
+
+        interpret(f, [])
+        assert not os.path.exists(self.ufilename)
+        assert os.path.exists(self.ufilename + '.new')
+
     def test_listdir(self):
         udir = UnicodeWithEncoding(os.path.dirname(self.ufilename))
 

From pypy.commits at gmail.com  Mon Jun 20 18:16:37 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge heads
Message-ID: <57686b45.820b1c0a.dc4e8.190b@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85269:b9b012bb9824
Date: 2016-05-22 14:44 -0400
http://bitbucket.org/pypy/pypy/changeset/b9b012bb9824/

Log:	Merge heads

diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -4526,6 +4526,10 @@
 
         # make sure we have an example of each type of descriptor
         for d, n in zip(descriptors, types):
+            if (support.check_impl_detail(pypy=True) and
+                n in ('method', 'member', 'wrapper')):
+                # PyPy doesn't have these
+                continue
             self.assertEqual(type(d).__name__, n + '_descriptor')
 
         for d in descriptors:
@@ -4539,7 +4543,7 @@
 
         class X:
             pass
-        with self.assertRaises(TypeError):
+        with self.assertRaises((AttributeError, TypeError)):
             del X.__qualname__
 
         self.assertRaises(TypeError, type.__dict__['__qualname__'].__set__,
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -277,7 +277,16 @@
     if StdErrPrinter is not None:
         sys.stderr = sys.__stderr__ = StdErrPrinter(2)
 
-    if 1:  # keep indentation
+    # Hack to avoid recursion issues during bootstrapping: pre-import
+    # the utf-8 and latin-1 codecs
+    encerr = None
+    try:
+        import encodings.utf_8
+        import encodings.latin_1
+    except ImportError as e:
+        encerr = e
+
+    try:
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
         else:
@@ -296,6 +305,10 @@
             print("Python error:  is a directory, cannot continue",
                   file=sys.stderr)
             os._exit(1)
+    finally:
+        if encerr:
+            display_exception(encerr)
+            del encerr
 
 def create_stdio(fd, writing, name, encoding, errors, unbuffered):
     import io
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1138,12 +1138,14 @@
             old_last_exception = self.last_exception
             self.last_exception = operr
             w_traceback = self.space.wrap(operr.get_traceback())
-            w_suppress = self.call_contextmanager_exit_function(
-                w_exitfunc,
-                operr.w_type,
-                operr.get_w_value(self.space),
-                w_traceback)
-            self.last_exception = old_last_exception
+            try:
+                w_suppress = self.call_contextmanager_exit_function(
+                    w_exitfunc,
+                    operr.w_type,
+                    operr.get_w_value(self.space),
+                    w_traceback)
+            finally:
+                self.last_exception = old_last_exception
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.
                 self.settopvalue(self.space.w_None)
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -439,7 +439,6 @@
             fail('No exception raised')
 
     def test_context_with_suppressed(self):
-        # XXX: requires with statement's WHY_SILENCED
         class RaiseExc:
             def __init__(self, exc):
                 self.exc = exc
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -60,6 +60,7 @@
     def test_descr_getsetproperty(self):
         from types import FrameType
         assert FrameType.f_lineno.__name__ == 'f_lineno'
+        assert FrameType.f_lineno.__qualname__ == 'frame.f_lineno'
         assert FrameType.f_lineno.__objclass__ is FrameType
         class A(object):
             pass
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -263,6 +263,7 @@
         self.doc = doc
         self.reqcls = cls
         self.name = ''
+        self.qualname = None
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
 
@@ -313,6 +314,21 @@
                 self.reqcls, Arguments(space, [w_obj,
                                                space.wrap(self.name)]))
 
+    def descr_get_qualname(self, space):
+        if self.qualname is None:
+            self.qualname = self._calculate_qualname(space)
+        return self.qualname
+
+    def _calculate_qualname(self, space):
+        if self.reqcls is None:
+            type_qualname = u'?'
+        else:
+            w_type = space.gettypeobject(self.reqcls.typedef)
+            type_qualname = space.unicode_w(
+                space.getattr(w_type, space.wrap('__qualname__')))
+        qualname = u"%s.%s" % (type_qualname, self.name.decode('utf-8'))
+        return space.wrap(qualname)
+
     def descr_get_objclass(space, property):
         return property.objclass_getter(space)
 
@@ -351,6 +367,7 @@
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
     __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __qualname__ = GetSetProperty(GetSetProperty.descr_get_qualname),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
     __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
     )
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -72,8 +72,8 @@
         'max'           : 'functional.max',
         'reversed'      : 'functional.W_ReversedIterator',
         'super'         : 'descriptor.W_Super',
-        'staticmethod'  : 'descriptor.StaticMethod',
-        'classmethod'   : 'descriptor.ClassMethod',
+        'staticmethod'  : 'pypy.interpreter.function.StaticMethod',
+        'classmethod'   : 'pypy.interpreter.function.ClassMethod',
         'property'      : 'descriptor.W_Property',
 
         'globals'       : 'interp_inspect.globals',
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -1,31 +1,41 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.function import StaticMethod, ClassMethod
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import (
-    TypeDef, interp_attrproperty_w, generic_new_descr, GetSetProperty)
+    GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty_w)
 from pypy.objspace.descroperation import object_getattribute
 
 
 class W_Super(W_Root):
-    def __init__(self, space, w_starttype, w_objtype, w_self):
+
+    def __init__(self, space):
+        self.w_starttype = None
+        self.w_objtype = None
+        self.w_self = None
+
+    def descr_init(self, space, w_starttype=None, w_obj_or_type=None):
+        if space.is_none(w_starttype):
+            w_starttype, w_obj_or_type = _super_from_frame(space)
+        if space.is_none(w_obj_or_type):
+            w_type = None  # unbound super object
+            w_obj_or_type = space.w_None
+        else:
+            w_type = _supercheck(space, w_starttype, w_obj_or_type)
         self.w_starttype = w_starttype
-        self.w_objtype = w_objtype
-        self.w_self = w_self
+        self.w_objtype = w_type
+        self.w_self = w_obj_or_type
 
     def get(self, space, w_obj, w_type=None):
-        w = space.wrap
         if self.w_self is None or space.is_w(w_obj, space.w_None):
-            return w(self)
+            return self
         else:
             # if type(self) is W_Super:
             #     XXX write a fast path for this common case
-            w_selftype = space.type(w(self))
+            w_selftype = space.type(self)
             return space.call_function(w_selftype, self.w_starttype, w_obj)
 
-    @unwrap_spec(name=str)
-    def getattribute(self, space, name):
-        w = space.wrap
+    def getattribute(self, space, w_name):
+        name = space.str_w(w_name)
         # only use a special logic for bound super objects and not for
         # getting the __class__ of the super object itself.
         if self.w_objtype is not None and name != '__class__':
@@ -45,73 +55,68 @@
                 return space.get_and_call_function(w_get, w_value,
                                                    w_obj, self.w_objtype)
         # fallback to object.__getattribute__()
-        return space.call_function(object_getattribute(space),
-                                   w(self), w(name))
+        return space.call_function(object_getattribute(space), self, w_name)
 
-def descr_new_super(space, w_subtype, w_starttype=None, w_obj_or_type=None):
-    if space.is_none(w_starttype):
-        # Call super(), without args -- fill in from __class__
-        # and first local variable on the stack.
-        ec = space.getexecutioncontext()
-        frame = ec.gettopframe()
-        code = frame.pycode
-        if not code:
-            raise oefmt(space.w_RuntimeError, "super(): no code object")
-        if code.co_argcount == 0:
-            raise oefmt(space.w_RuntimeError, "super(): no arguments")
-        w_obj = frame.locals_cells_stack_w[0]
-        if not w_obj:
-            raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
-        index = 0
-        for name in code.co_freevars:
-            if name == "__class__":
-                break
-            index += 1
-        else:
-            raise oefmt(space.w_RuntimeError,
-                        "super(): __class__ cell not found")
-        # a kind of LOAD_DEREF
-        cell = frame._getcell(len(code.co_cellvars) + index)
-        try:
-            w_starttype = cell.get()
-        except ValueError:
-            raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
-        w_obj_or_type = w_obj
+def _super_from_frame(space):
+    """super() without args -- fill in from __class__ and first local
+    variable on the stack.
+    """
+    frame = space.getexecutioncontext().gettopframe()
+    code = frame.pycode
+    if not code:
+        raise oefmt(space.w_RuntimeError, "super(): no code object")
+    if code.co_argcount == 0:
+        raise oefmt(space.w_RuntimeError, "super(): no arguments")
+    w_obj = frame.locals_cells_stack_w[0]
+    if not w_obj:
+        raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
+    for index, name in enumerate(code.co_freevars):
+        if name == "__class__":
+            break
+    else:
+        raise oefmt(space.w_RuntimeError, "super(): __class__ cell not found")
+    # a kind of LOAD_DEREF
+    cell = frame._getcell(len(code.co_cellvars) + index)
+    try:
+        w_starttype = cell.get()
+    except ValueError:
+        raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
+    return w_starttype, w_obj
 
-    if space.is_none(w_obj_or_type):
-        w_type = None  # unbound super object
-        w_obj_or_type = space.w_None
-    else:
-        w_objtype = space.type(w_obj_or_type)
-        if space.is_true(space.issubtype(w_objtype, space.w_type)) and \
-            space.is_true(space.issubtype(w_obj_or_type, w_starttype)):
-            w_type = w_obj_or_type # special case for class methods
-        elif space.is_true(space.issubtype(w_objtype, w_starttype)):
-            w_type = w_objtype # normal case
-        else:
-            try:
-                w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
-            except OperationError as o:
-                if not o.match(space, space.w_AttributeError):
-                    raise
-                w_type = w_objtype
-            if not space.is_true(space.issubtype(w_type, w_starttype)):
-                raise oefmt(space.w_TypeError,
-                            "super(type, obj): obj must be an instance or "
-                            "subtype of type")
-    # XXX the details of how allocate_instance() should be used are not
-    # really well defined
-    w_result = space.allocate_instance(W_Super, w_subtype)
-    W_Super.__init__(w_result, space, w_starttype, w_type, w_obj_or_type)
-    return w_result
+def _supercheck(space, w_starttype, w_obj_or_type):
+    """Check that the super() call makes sense. Returns a type"""
+    w_objtype = space.type(w_obj_or_type)
+
+    if (space.is_true(space.issubtype(w_objtype, space.w_type)) and
+        space.is_true(space.issubtype(w_obj_or_type, w_starttype))):
+        # special case for class methods
+        return w_obj_or_type
+
+    if space.is_true(space.issubtype(w_objtype, w_starttype)):
+        # normal case
+        return w_objtype
+
+    try:
+        w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
+    except OperationError as e:
+        if not e.match(space, space.w_AttributeError):
+            raise
+        w_type = w_objtype
+
+    if space.is_true(space.issubtype(w_type, w_starttype)):
+        return w_type
+    raise oefmt(space.w_TypeError,
+                "super(type, obj): obj must be an instance or subtype of type")
 
 W_Super.typedef = TypeDef(
     'super',
-    __new__          = interp2app(descr_new_super),
+    __new__          = generic_new_descr(W_Super),
+    __init__         = interp2app(W_Super.descr_init),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
-    __doc__          =     """super(type) -> unbound super object
+    __doc__          =     """\
+super(type) -> unbound super object
 super(type, obj) -> bound super object; requires isinstance(obj, type)
 super(type, type2) -> bound super object; requires issubclass(type2, type)
 
@@ -129,10 +134,10 @@
     def __init__(self, space):
         pass
 
-    @unwrap_spec(w_fget = WrappedDefault(None),
-                 w_fset = WrappedDefault(None),
-                 w_fdel = WrappedDefault(None),
-                 w_doc = WrappedDefault(None))
+    @unwrap_spec(w_fget=WrappedDefault(None),
+                 w_fset=WrappedDefault(None),
+                 w_fdel=WrappedDefault(None),
+                 w_doc=WrappedDefault(None))
     def init(self, space, w_fget=None, w_fset=None, w_fdel=None, w_doc=None):
         self.w_fget = w_fget
         self.w_fset = w_fset
@@ -142,18 +147,17 @@
         # our __doc__ comes from the getter if we don't have an explicit one
         if (space.is_w(self.w_doc, space.w_None) and
             not space.is_w(self.w_fget, space.w_None)):
-            w_getter_doc = space.findattr(self.w_fget, space.wrap("__doc__"))
+            w_getter_doc = space.findattr(self.w_fget, space.wrap('__doc__'))
             if w_getter_doc is not None:
                 if type(self) is W_Property:
                     self.w_doc = w_getter_doc
                 else:
-                    space.setattr(space.wrap(self), space.wrap("__doc__"),
-                                  w_getter_doc)
+                    space.setattr(self, space.wrap('__doc__'), w_getter_doc)
                 self.getter_doc = True
 
     def get(self, space, w_obj, w_objtype=None):
         if space.is_w(w_obj, space.w_None):
-            return space.wrap(self)
+            return self
         if space.is_w(self.w_fget, space.w_None):
             raise oefmt(space.w_AttributeError, "unreadable attribute")
         return space.call_function(self.w_fget, w_obj)
@@ -191,7 +195,8 @@
         else:
             w_doc = self.w_doc
         w_type = self.getclass(space)
-        return space.call_function(w_type, w_getter, w_setter, w_deleter, w_doc)
+        return space.call_function(w_type, w_getter, w_setter, w_deleter,
+                                   w_doc)
 
     def descr_isabstract(self, space):
         return space.newbool(space.isabstractmethod_w(self.w_fget) or
@@ -200,7 +205,8 @@
 
 W_Property.typedef = TypeDef(
     'property',
-    __doc__ = '''property(fget=None, fset=None, fdel=None, doc=None) -> property attribute
+    __doc__ = '''\
+property(fget=None, fset=None, fdel=None, doc=None) -> property attribute
 
 fget is a function to be used for getting an attribute value, and likewise
 fset is a function for setting, and fdel a function for deleting, an
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -233,10 +233,9 @@
     # __________ app-level attributes __________
     def dir(self):
         space = self.space
-        w_self = space.wrap(self)
         lst = [space.wrap(name)
                   for name in _name_of_attributes
-                  if space.findattr(w_self, space.wrap(name)) is not None]
+                  if space.findattr(self, space.wrap(name)) is not None]
         return space.newlist(lst)
 
     def _fget(self, attrchar):
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -141,8 +141,6 @@
 
 class AppTestPartialEvaluation:
     spaceconfig = dict(usemodules=['array',])
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_winreg')
 
     def test_partial_utf8(self):
         import _codecs
@@ -767,7 +765,7 @@
         try:
             # test for non-latin1 codepage, more general test needed
             import winreg
-            key = winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE,
+            key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
                         r'System\CurrentControlSet\Control\Nls\CodePage')
             if winreg.QueryValueEx(key, 'ACP')[0] == u'1255':  # non-latin1
                 toencode = u'caf\xbf',b'caf\xbf'
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -389,20 +389,18 @@
     def copy(self):
         "Return a shallow copy of a deque."
         space = self.space
-        w_self = space.wrap(self)
         if self.maxlen == sys.maxint:
-            return space.call_function(space.type(w_self), w_self)
+            return space.call_function(space.type(self), self)
         else:
-            return space.call_function(space.type(w_self), w_self,
+            return space.call_function(space.type(self), self,
                                        space.wrap(self.maxlen))
 
     def reduce(self):
         "Return state information for pickling."
         space = self.space
-        w_self = space.wrap(self)
-        w_type = space.type(w_self)
-        w_dict = space.findattr(w_self, space.wrap('__dict__'))
-        w_list = space.call_function(space.w_list, w_self)
+        w_type = space.type(self)
+        w_dict = space.findattr(self, space.wrap('__dict__'))
+        w_list = space.call_function(space.w_list, self)
         if w_dict is None:
             if self.maxlen == sys.maxint:
                 result = [
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -156,12 +156,12 @@
 
 
 class W_WeakrefBase(W_Root):
-    def __init__(w_self, space, w_obj, w_callable):
+    def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
-        w_self.space = space
+        self.space = space
         assert w_obj is not None
-        w_self.w_obj_weak = weakref.ref(w_obj)
-        w_self.w_callable = w_callable
+        self.w_obj_weak = weakref.ref(w_obj)
+        self.w_callable = w_callable
 
     @jit.dont_look_inside
     def dereference(self):
@@ -171,8 +171,8 @@
     def clear(self):
         self.w_obj_weak = dead_ref
 
-    def activate_callback(w_self):
-        w_self.space.call_function(w_self.w_callable, w_self)
+    def activate_callback(self):
+        self.space.call_function(self.w_callable, self)
 
     def descr__repr__(self, space):
         w_obj = self.dereference()
@@ -189,9 +189,9 @@
 
 
 class W_Weakref(W_WeakrefBase):
-    def __init__(w_self, space, w_obj, w_callable):
-        W_WeakrefBase.__init__(w_self, space, w_obj, w_callable)
-        w_self.w_hash = None
+    def __init__(self, space, w_obj, w_callable):
+        W_WeakrefBase.__init__(self, space, w_obj, w_callable)
+        self.w_hash = None
 
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -19,7 +19,7 @@
     canSaveKey = True
 
 class AppTestHKey:
-    spaceconfig = dict(usemodules=('_winreg',))
+    #spaceconfig = dict(usemodules=('_winreg',))
 
     def test_repr(self):
         import winreg
@@ -27,7 +27,7 @@
         assert str(k) == ""
 
 class AppTestFfi:
-    spaceconfig = dict(usemodules=('_winreg',))
+    #spaceconfig = dict(usemodules=('_winreg',))
 
     def setup_class(cls):
         import _winreg
@@ -53,9 +53,9 @@
         w_test_data.append(w_btest)
 
     def teardown_class(cls):
-        import _winreg
+        import winreg
         try:
-            _winreg.DeleteKey(cls.root_key, cls.test_key_name)
+            winreg.DeleteKey(cls.root_key, cls.test_key_name)
         except WindowsError:
             pass
 
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -349,7 +349,7 @@
     compress = interp2app(W_BZ2Compressor.compress),
     flush = interp2app(W_BZ2Compressor.flush),
 )
-
+W_BZ2Compressor.typedef.acceptable_as_base_class = False
 
 def descr_decompressor__new__(space, w_subtype):
     x = space.allocate_instance(W_BZ2Decompressor, w_subtype)
@@ -457,3 +457,4 @@
     eof = GetSetProperty(W_BZ2Decompressor.eof_w),
     decompress = interp2app(W_BZ2Decompressor.decompress),
 )
+W_BZ2Decompressor.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -54,8 +54,6 @@
         st_flags = structseqfield(23, "user defined flags for file")
 
     def __init__(self, *args, **kw):
-        super(stat_result, self).__init__(*args, **kw)
-
         # If we have been initialized from a tuple,
         # st_?time might be set to None. Initialize it
         # from the int slots.
diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py
--- a/pypy/module/thread/os_local.py
+++ b/pypy/module/thread/os_local.py
@@ -50,10 +50,9 @@
         self.dicts[ec] = w_dict
         # call __init__
         try:
-            w_self = space.wrap(self)
-            w_type = space.type(w_self)
+            w_type = space.type(self)
             w_init = space.getattr(w_type, space.wrap("__init__"))
-            space.call_obj_args(w_init, w_self, self.initargs)
+            space.call_obj_args(w_init, self, self.initargs)
         except:
             # failed, forget w_dict and propagate the exception
             del self.dicts[ec]
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -364,6 +364,28 @@
         space = self.space
         return space.wrap_fsdecoded(self.filename)
 
+    def _find_loader(self, space, fullname):
+        filename = self.make_filename(fullname)
+        for _, _, ext in ENUMERATE_EXTS:
+            if self.have_modulefile(space, filename + ext):
+                return True, None
+        # See if this is a directory (part of a namespace pkg)
+        dirpath = self.prefix + fullname
+        if self.have_modulefile(space, dirpath + ZIPSEP):
+            return True, self.filename + os.path.sep + self.corr_zname(dirpath)
+        return False, None
+
+    @unwrap_spec(fullname='str0')
+    def find_loader(self, space, fullname, w_path=None):
+        found, ns_portion = self._find_loader(space, fullname)
+        if not found:
+            result = [space.w_None, space.newlist([])]
+        elif not ns_portion:
+            result = [self, space.newlist([])]
+        else:
+            result = [space.w_None, space.newlist([space.wrap(ns_portion)])]
+        return space.newtuple(result)
+
 def descr_new_zipimporter(space, w_type, w_name):
     name = space.fsencode_w(w_name)
     ok = False
@@ -422,6 +444,7 @@
     get_filename = interp2app(W_ZipImporter.get_filename),
     is_package  = interp2app(W_ZipImporter.is_package),
     load_module = interp2app(W_ZipImporter.load_module),
+    find_loader = interp2app(W_ZipImporter.find_loader),
     archive     = GetSetProperty(W_ZipImporter.getarchive),
     prefix      = GetSetProperty(W_ZipImporter.getprefix),
 )
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -440,6 +440,12 @@
         self.writefile('x1test/__init__.py', 'raise ValueError')
         raises(ValueError, __import__, 'x1test', None, None, [])
 
+    def test_namespace_pkg(self):
+        self.writefile('foo/', '')
+        self.writefile('foo/one.py', "attr = 'portion1 foo one'\n")
+        foo = __import__('foo.one', None, None, [])
+        assert foo.one.attr == 'portion1 foo one'
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):
diff --git a/pypy/objspace/std/noneobject.py b/pypy/objspace/std/noneobject.py
--- a/pypy/objspace/std/noneobject.py
+++ b/pypy/objspace/std/noneobject.py
@@ -4,7 +4,7 @@
 
 
 class W_NoneObject(W_Root):
-    def unwrap(w_self, space):
+    def unwrap(self, space):
         return None
 
     @staticmethod
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -84,23 +84,23 @@
     'object()' call."""
 
 
+def _excess_args(__args__):
+    return bool(__args__.arguments_w) or bool(__args__.keywords)
+
 def descr__new__(space, w_type, __args__):
-    from pypy.objspace.std.objectobject import W_ObjectObject
     from pypy.objspace.std.typeobject import _precheck_for_new
+    w_type = _precheck_for_new(space, w_type)
+
     # don't allow arguments if the default object.__init__() is about
     # to be called
-    w_type = _precheck_for_new(space, w_type)
-    w_parentinit, _ = w_type.lookup_where('__init__')
-    if w_parentinit is space.w_object:
-        try:
-            __args__.fixedunpack(0)
-        except ValueError:
+    if _excess_args(__args__):
+        w_parent_init, _ = space.lookup_in_type_where(w_type, '__init__')
+        if w_parent_init is space.w_object:
             raise oefmt(space.w_TypeError,
-                        "default __new__ takes no parameters")
+                        "object() takes no parameters")
     if w_type.is_abstract():
         _abstract_method_error(space, w_type)
-    w_obj = space.allocate_instance(W_ObjectObject, w_type)
-    return w_obj
+    return space.allocate_instance(W_ObjectObject, w_type)
 
 
 def descr___subclasshook__(space, __args__):
@@ -109,12 +109,10 @@
 
 def descr__init__(space, w_obj, __args__):
     # don't allow arguments unless __new__ is overridden
-    w_type = space.type(w_obj)
-    w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
-    if w_parent_new is space.w_object:
-        try:
-            __args__.fixedunpack(0)
-        except ValueError:
+    if _excess_args(__args__):
+        w_type = space.type(w_obj)
+        w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
+        if w_parent_new is space.w_object:
             raise oefmt(space.w_TypeError,
                         "object.__init__() takes no parameters")
 
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -12,13 +12,13 @@
 class W_SliceObject(W_Root):
     _immutable_fields_ = ['w_start', 'w_stop', 'w_step']
 
-    def __init__(w_self, w_start, w_stop, w_step):
+    def __init__(self, w_start, w_stop, w_step):
         assert w_start is not None
         assert w_stop is not None
         assert w_step is not None
-        w_self.w_start = w_start
-        w_self.w_stop = w_stop
-        w_self.w_step = w_step
+        self.w_start = w_start
+        self.w_stop = w_stop
+        self.w_step = w_step
 
     def unwrap(w_slice, space):
         return slice(space.unwrap(w_slice.w_start), space.unwrap(w_slice.w_stop), space.unwrap(w_slice.w_step))
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -26,10 +26,10 @@
         else:
             return self.w_str._value
 
-    def __repr__(w_self):
+    def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%r[:%d])" % (
-            w_self.__class__.__name__, w_self.builder, w_self.length)
+            self.__class__.__name__, self.builder, self.length)
 
     def unwrap(self, space):
         return self.force()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -154,220 +154,220 @@
     w_new_function = None
 
     @dont_look_inside
-    def __init__(w_self, space, name, bases_w, dict_w,
+    def __init__(self, space, name, bases_w, dict_w,
                  overridetypedef=None, force_new_layout=False):
-        w_self.space = space
-        w_self.name = name
-        w_self.qualname = None
-        w_self.bases_w = bases_w
-        w_self.dict_w = dict_w
-        w_self.hasdict = False
-        w_self.hasuserdel = False
-        w_self.weakrefable = False
-        w_self.w_doc = space.w_None
-        w_self.weak_subclasses = []
-        w_self.flag_heaptype = False
-        w_self.flag_cpytype = False
-        w_self.flag_abstract = False
-        w_self.flag_sequence_bug_compat = False
-        w_self.flag_map_or_seq = '?'   # '?' means "don't know, check otherwise"
+        self.space = space
+        self.name = name
+        self.qualname = None
+        self.bases_w = bases_w
+        self.dict_w = dict_w
+        self.hasdict = False
+        self.hasuserdel = False
+        self.weakrefable = False
+        self.w_doc = space.w_None
+        self.weak_subclasses = []
+        self.flag_heaptype = False
+        self.flag_cpytype = False
+        self.flag_abstract = False
+        self.flag_sequence_bug_compat = False
+        self.flag_map_or_seq = '?'   # '?' means "don't know, check otherwise"
 
         if overridetypedef is not None:
             assert not force_new_layout
-            layout = setup_builtin_type(w_self, overridetypedef)
+            layout = setup_builtin_type(self, overridetypedef)
         else:
-            layout = setup_user_defined_type(w_self, force_new_layout)
-        w_self.layout = layout
+            layout = setup_user_defined_type(self, force_new_layout)
+        self.layout = layout
 
-        if not is_mro_purely_of_types(w_self.mro_w):
+        if not is_mro_purely_of_types(self.mro_w):
             pass
         else:
             # the _version_tag should change, whenever the content of
             # dict_w of any of the types in the mro changes, or if the mro
             # itself changes
-            w_self._version_tag = VersionTag()
+            self._version_tag = VersionTag()
         from pypy.objspace.std.mapdict import DictTerminator, NoDictTerminator
         # if the typedef has a dict, then the rpython-class does all the dict
         # management, which means from the point of view of mapdict there is no
         # dict.
-        typedef = w_self.layout.typedef
-        if (w_self.hasdict and not typedef.hasdict):
-            w_self.terminator = DictTerminator(space, w_self)
+        typedef = self.layout.typedef
+        if (self.hasdict and not typedef.hasdict):
+            self.terminator = DictTerminator(space, self)
         else:
-            w_self.terminator = NoDictTerminator(space, w_self)
+            self.terminator = NoDictTerminator(space, self)
 
     def __repr__(self):
         "NOT_RPYTHON"
         return '' % (self.name, id(self))
 
-    def mutated(w_self, key):
+    def mutated(self, key):
         """
         The type is being mutated. key is either the string containing the
         specific attribute which is being deleted/set or None to indicate a
         generic mutation.
         """
-        space = w_self.space
-        assert w_self.is_heaptype() or w_self.is_cpytype()
+        space = self.space
+        assert self.is_heaptype() or self.is_cpytype()
 
-        w_self.uses_object_getattribute = False
+        self.uses_object_getattribute = False
         # ^^^ conservative default, fixed during real usage
 
         if (key is None or key == '__eq__' or key == '__hash__'):
-            w_self.compares_by_identity_status = UNKNOWN
+            self.compares_by_identity_status = UNKNOWN
 
         if space.config.objspace.std.newshortcut:
-            w_self.w_new_function = None
+            self.w_new_function = None
 
-        if w_self._version_tag is not None:
-            w_self._version_tag = VersionTag()
+        if self._version_tag is not None:
+            self._version_tag = VersionTag()
 
-        subclasses_w = w_self.get_subclasses()
+        subclasses_w = self.get_subclasses()
         for w_subclass in subclasses_w:
             assert isinstance(w_subclass, W_TypeObject)
             w_subclass.mutated(key)
 
-    def version_tag(w_self):
-        if not we_are_jitted() or w_self.is_heaptype():
-            return w_self._version_tag
+    def version_tag(self):
+        if not we_are_jitted() or self.is_heaptype():
+            return self._version_tag
         # prebuilt objects cannot get their version_tag changed
-        return w_self._pure_version_tag()
+        return self._pure_version_tag()
 
     @elidable_promote()
-    def _pure_version_tag(w_self):
-        return w_self._version_tag
+    def _pure_version_tag(self):
+        return self._version_tag
 
-    def getattribute_if_not_from_object(w_self):
+    def getattribute_if_not_from_object(self):
         """ this method returns the applevel __getattribute__ if that is not
         the one from object, in which case it returns None """
         from pypy.objspace.descroperation import object_getattribute
         if not we_are_jitted():
-            if not w_self.uses_object_getattribute:
+            if not self.uses_object_getattribute:
                 # slow path: look for a custom __getattribute__ on the class
-                w_descr = w_self.lookup('__getattribute__')
+                w_descr = self.lookup('__getattribute__')
                 # if it was not actually overriden in the class, we remember this
                 # fact for the next time.
-                if w_descr is object_getattribute(w_self.space):
-                    w_self.uses_object_getattribute = True
+                if w_descr is object_getattribute(self.space):
+                    self.uses_object_getattribute = True
                 else:
                     return w_descr
             return None
         # in the JIT case, just use a lookup, because it is folded away
         # correctly using the version_tag
-        w_descr = w_self.lookup('__getattribute__')
-        if w_descr is not object_getattribute(w_self.space):
+        w_descr = self.lookup('__getattribute__')
+        if w_descr is not object_getattribute(self.space):
             return w_descr
 
-    def has_object_getattribute(w_self):
-        return w_self.getattribute_if_not_from_object() is None
+    def has_object_getattribute(self):
+        return self.getattribute_if_not_from_object() is None
 
-    def compares_by_identity(w_self):
+    def compares_by_identity(self):
         from pypy.objspace.descroperation import object_hash, type_eq
         #
-        if w_self.compares_by_identity_status != UNKNOWN:
+        if self.compares_by_identity_status != UNKNOWN:
             # fast path
-            return w_self.compares_by_identity_status == COMPARES_BY_IDENTITY
+            return self.compares_by_identity_status == COMPARES_BY_IDENTITY
         #
-        default_hash = object_hash(w_self.space)
-        my_eq = w_self.lookup('__eq__')
-        overrides_eq = (my_eq and my_eq is not type_eq(w_self.space))
+        default_hash = object_hash(self.space)
+        my_eq = self.lookup('__eq__')
+        overrides_eq = (my_eq and my_eq is not type_eq(self.space))
         overrides_eq_cmp_or_hash = (overrides_eq or
-                                    w_self.lookup('__hash__') is not default_hash)
+                                    self.lookup('__hash__') is not default_hash)
         if overrides_eq_cmp_or_hash:
-            w_self.compares_by_identity_status = OVERRIDES_EQ_CMP_OR_HASH
+            self.compares_by_identity_status = OVERRIDES_EQ_CMP_OR_HASH
         else:
-            w_self.compares_by_identity_status = COMPARES_BY_IDENTITY
-        return w_self.compares_by_identity_status == COMPARES_BY_IDENTITY
+            self.compares_by_identity_status = COMPARES_BY_IDENTITY
+        return self.compares_by_identity_status == COMPARES_BY_IDENTITY
 
-    def ready(w_self):
-        for w_base in w_self.bases_w:
+    def ready(self):
+        for w_base in self.bases_w:
             if not isinstance(w_base, W_TypeObject):
                 continue
-            w_base.add_subclass(w_self)
+            w_base.add_subclass(self)
 
     # compute a tuple that fully describes the instance layout
-    def get_full_instance_layout(w_self):
-        layout = w_self.layout
-        return (layout, w_self.hasdict, w_self.weakrefable)
+    def get_full_instance_layout(self):
+        layout = self.layout
+        return (layout, self.hasdict, self.weakrefable)
 
-    def compute_default_mro(w_self):
-        return compute_C3_mro(w_self.space, w_self)
+    def compute_default_mro(self):
+        return compute_C3_mro(self.space, self)
 
-    def getdictvalue(w_self, space, attr):
-        version_tag = w_self.version_tag()
+    def getdictvalue(self, space, attr):
+        version_tag = self.version_tag()
         if version_tag is not None:
             return unwrap_cell(
                 space,
-                w_self._pure_getdictvalue_no_unwrapping(
+                self._pure_getdictvalue_no_unwrapping(
                     space, version_tag, attr))
-        w_value = w_self._getdictvalue_no_unwrapping(space, attr)
+        w_value = self._getdictvalue_no_unwrapping(space, attr)
         return unwrap_cell(space, w_value)
 
-    def _getdictvalue_no_unwrapping(w_self, space, attr):
-        w_value = w_self.dict_w.get(attr, None)
-        if w_self.lazyloaders and w_value is None:
-            if attr in w_self.lazyloaders:
+    def _getdictvalue_no_unwrapping(self, space, attr):
+        w_value = self.dict_w.get(attr, None)
+        if self.lazyloaders and w_value is None:
+            if attr in self.lazyloaders:
                 # very clever next line: it forces the attr string
                 # to be interned.
                 space.new_interned_str(attr)
-                loader = w_self.lazyloaders[attr]
-                del w_self.lazyloaders[attr]
+                loader = self.lazyloaders[attr]
+                del self.lazyloaders[attr]
                 w_value = loader()
                 if w_value is not None:   # None means no such attribute
-                    w_self.dict_w[attr] = w_value
+                    self.dict_w[attr] = w_value
                     return w_value
         return w_value
 
     @elidable
-    def _pure_getdictvalue_no_unwrapping(w_self, space, version_tag, attr):
-        return w_self._getdictvalue_no_unwrapping(space, attr)
+    def _pure_getdictvalue_no_unwrapping(self, space, version_tag, attr):
+        return self._getdictvalue_no_unwrapping(space, attr)
 
-    def setdictvalue(w_self, space, name, w_value):
-        if not w_self.is_heaptype():
+    def setdictvalue(self, space, name, w_value):
+        if not self.is_heaptype():
             raise oefmt(space.w_TypeError,
-                        "can't set attributes on type object '%N'", w_self)
-        if name == "__del__" and name not in w_self.dict_w:
+                        "can't set attributes on type object '%N'", self)
+        if name == "__del__" and name not in self.dict_w:
             msg = ("a __del__ method added to an existing type will not be "
                    "called")
             space.warn(space.wrap(msg), space.w_RuntimeWarning)
-        version_tag = w_self.version_tag()
+        version_tag = self.version_tag()
         if version_tag is not None:
-            w_curr = w_self._pure_getdictvalue_no_unwrapping(
+            w_curr = self._pure_getdictvalue_no_unwrapping(
                     space, version_tag, name)
             w_value = write_cell(space, w_curr, w_value)
             if w_value is None:
                 return True
-        w_self.mutated(name)
-        w_self.dict_w[name] = w_value
+        self.mutated(name)
+        self.dict_w[name] = w_value
         return True
 
-    def deldictvalue(w_self, space, key):
-        if w_self.lazyloaders:
-            w_self._cleanup_()    # force un-lazification
-        if not w_self.is_heaptype():
+    def deldictvalue(self, space, key):
+        if self.lazyloaders:
+            self._cleanup_()    # force un-lazification
+        if not self.is_heaptype():
             raise oefmt(space.w_TypeError,
-                        "can't delete attributes on type object '%N'", w_self)
+                        "can't delete attributes on type object '%N'", self)
         try:
-            del w_self.dict_w[key]
+            del self.dict_w[key]
         except KeyError:
             return False
         else:
-            w_self.mutated(key)
+            self.mutated(key)
             return True
 
-    def lookup(w_self, name):
+    def lookup(self, name):
         # note that this doesn't call __get__ on the result at all
-        space = w_self.space
-        return w_self.lookup_where_with_method_cache(name)[1]
+        space = self.space
+        return self.lookup_where_with_method_cache(name)[1]
 
-    def lookup_where(w_self, name):
-        space = w_self.space
-        return w_self.lookup_where_with_method_cache(name)
+    def lookup_where(self, name):
+        space = self.space
+        return self.lookup_where_with_method_cache(name)
 
     @unroll_safe
-    def lookup_starting_at(w_self, w_starttype, name):
-        space = w_self.space
+    def lookup_starting_at(self, w_starttype, name):
+        space = self.space
         look = False
-        for w_class in w_self.mro_w:
+        for w_class in self.mro_w:
             if w_class is w_starttype:
                 look = True
             elif look:
@@ -377,54 +377,54 @@
         return None
 
     @unroll_safe
-    def _lookup(w_self, key):
+    def _lookup(self, key):
         # nowadays, only called from ../../tool/ann_override.py
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             w_value = w_class.getdictvalue(space, key)
             if w_value is not None:
                 return w_value
         return None
 
     @unroll_safe
-    def _lookup_where(w_self, key):
+    def _lookup_where(self, key):
         # like _lookup() but also returns the parent class in which the
         # attribute was found
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             w_value = w_class.getdictvalue(space, key)
             if w_value is not None:
                 return w_class, w_value
         return None, None
 
-    def _lookup_where_all_typeobjects(w_self, key):
-        # like _lookup_where(), but when we know that w_self.mro_w only
+    def _lookup_where_all_typeobjects(self, key):
+        # like _lookup_where(), but when we know that self.mro_w only
         # contains W_TypeObjects.  (It differs from _lookup_where() mostly
         # from a JIT point of view: it cannot invoke arbitrary Python code.)
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             assert isinstance(w_class, W_TypeObject)
             w_value = w_class._getdictvalue_no_unwrapping(space, key)
             if w_value is not None:
                 return w_class, w_value
         return None, None
 
-    def lookup_where_with_method_cache(w_self, name):
-        space = w_self.space
-        promote(w_self)
-        version_tag = promote(w_self.version_tag())
+    def lookup_where_with_method_cache(self, name):
+        space = self.space
+        promote(self)
+        version_tag = promote(self.version_tag())
         if version_tag is None:
-            tup = w_self._lookup_where(name)
+            tup = self._lookup_where(name)
             return tup
-        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        tup_w = self._pure_lookup_where_with_method_cache(name, version_tag)
         w_class, w_value = tup_w
         if isinstance(w_value, MutableCell):
             return w_class, w_value.unwrap_cell(space)
         return tup_w   # don't make a new tuple, reuse the old one
 
     @elidable
-    def _pure_lookup_where_with_method_cache(w_self, name, version_tag):
-        space = w_self.space
+    def _pure_lookup_where_with_method_cache(self, name, version_tag):
+        space = self.space
         cache = space.fromcache(MethodCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp
         SHIFT1 = SHIFT2 - 5
@@ -449,70 +449,70 @@
                 tup = cache.lookup_where[method_hash]
                 if space.config.objspace.std.withmethodcachecounter:
                     cache.hits[name] = cache.hits.get(name, 0) + 1
-#                print "hit", w_self, name
+#                print "hit", self, name
                 return tup
-        tup = w_self._lookup_where_all_typeobjects(name)
+        tup = self._lookup_where_all_typeobjects(name)
         cache.versions[method_hash] = version_tag
         cache.names[method_hash] = name
         cache.lookup_where[method_hash] = tup
         if space.config.objspace.std.withmethodcachecounter:
             cache.misses[name] = cache.misses.get(name, 0) + 1
-#        print "miss", w_self, name
+#        print "miss", self, name
         return tup
 
-    def check_user_subclass(w_self, w_subtype):
-        space = w_self.space
+    def check_user_subclass(self, w_subtype):
+        space = self.space
         if not isinstance(w_subtype, W_TypeObject):
             raise oefmt(space.w_TypeError,
                         "X is not a type object ('%T')", w_subtype)
-        if not w_subtype.issubtype(w_self):
+        if not w_subtype.issubtype(self):
             raise oefmt(space.w_TypeError,
                         "%N.__new__(%N): %N is not a subtype of %N",
-                        w_self, w_subtype, w_subtype, w_self)
-        if w_self.layout.typedef is not w_subtype.layout.typedef:
+                        self, w_subtype, w_subtype, self)
+        if self.layout.typedef is not w_subtype.layout.typedef:
             raise oefmt(space.w_TypeError,
                         "%N.__new__(%N) is not safe, use %N.__new__()",
-                        w_self, w_subtype, w_subtype)
+                        self, w_subtype, w_subtype)
         return w_subtype
 
-    def _cleanup_(w_self):
+    def _cleanup_(self):
         "NOT_RPYTHON.  Forces the lazy attributes to be computed."
-        if 'lazyloaders' in w_self.__dict__:
-            for attr in w_self.lazyloaders.keys():
-                w_self.getdictvalue(w_self.space, attr)
-            del w_self.lazyloaders
+        if 'lazyloaders' in self.__dict__:
+            for attr in self.lazyloaders.keys():
+                self.getdictvalue(self.space, attr)
+            del self.lazyloaders
 
-    def getdict(w_self, space): # returning a dict-proxy!
+    def getdict(self, space): # returning a dict-proxy!
         from pypy.objspace.std.dictproxyobject import DictProxyStrategy
         from pypy.objspace.std.dictproxyobject import W_DictProxyObject
-        if w_self.lazyloaders:
-            w_self._cleanup_()    # force un-lazification
+        if self.lazyloaders:
+            self._cleanup_()    # force un-lazification
         strategy = space.fromcache(DictProxyStrategy)
-        storage = strategy.erase(w_self)
+        storage = strategy.erase(self)
         return W_DictProxyObject(space, strategy, storage)
 
-    def is_heaptype(w_self):
-        return w_self.flag_heaptype
+    def is_heaptype(self):
+        return self.flag_heaptype
 
-    def is_cpytype(w_self):
-        return w_self.flag_cpytype
+    def is_cpytype(self):
+        return self.flag_cpytype
 
-    def is_abstract(w_self):
-        return w_self.flag_abstract
+    def is_abstract(self):
+        return self.flag_abstract
 
-    def set_abstract(w_self, abstract):
-        w_self.flag_abstract = bool(abstract)
+    def set_abstract(self, abstract):
+        self.flag_abstract = bool(abstract)
 
-    def issubtype(w_self, w_type):
-        promote(w_self)
+    def issubtype(self, w_type):
+        promote(self)
         promote(w_type)
         if we_are_jitted():
-            version_tag1 = w_self.version_tag()
+            version_tag1 = self.version_tag()
             version_tag2 = w_type.version_tag()
             if version_tag1 is not None and version_tag2 is not None:
-                res = _pure_issubtype(w_self, w_type, version_tag1, version_tag2)
+                res = _pure_issubtype(self, w_type, version_tag1, version_tag2)
                 return res
-        return _issubtype(w_self, w_type)
+        return _issubtype(self, w_type)
 
     def get_module(self):
         space = self.space
@@ -540,8 +540,8 @@
     def getqualname(self, space):
         return self.qualname or self.getname(space)
 
-    def add_subclass(w_self, w_subclass):
-        space = w_self.space
+    def add_subclass(self, w_subclass):
+        space = self.space
         if not space.config.translation.rweakref:
             # We don't have weakrefs!  In this case, every class stores
             # subclasses in a non-weak list.  ALL CLASSES LEAK!  To make
@@ -554,26 +554,26 @@
 
         assert isinstance(w_subclass, W_TypeObject)
         newref = weakref.ref(w_subclass)
-        for i in range(len(w_self.weak_subclasses)):
-            ref = w_self.weak_subclasses[i]
+        for i in range(len(self.weak_subclasses)):
+            ref = self.weak_subclasses[i]
             if ref() is None:
-                w_self.weak_subclasses[i] = newref
+                self.weak_subclasses[i] = newref
                 return
         else:
-            w_self.weak_subclasses.append(newref)
+            self.weak_subclasses.append(newref)
 
-    def remove_subclass(w_self, w_subclass):
-        space = w_self.space
-        for i in range(len(w_self.weak_subclasses)):
-            ref = w_self.weak_subclasses[i]
+    def remove_subclass(self, w_subclass):
+        space = self.space
+        for i in range(len(self.weak_subclasses)):
+            ref = self.weak_subclasses[i]
             if ref() is w_subclass:
-                del w_self.weak_subclasses[i]
+                del self.weak_subclasses[i]
                 return
 
-    def get_subclasses(w_self):
-        space = w_self.space
+    def get_subclasses(self):
+        space = self.space
         subclasses_w = []
-        for ref in w_self.weak_subclasses:
+        for ref in self.weak_subclasses:
             w_ob = ref()
             if w_ob is not None:
                 subclasses_w.append(w_ob)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -26,23 +26,23 @@
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value', '_utf8?']
 
-    def __init__(w_self, unistr):
+    def __init__(self, unistr):
         assert isinstance(unistr, unicode)
-        w_self._value = unistr
-        w_self._utf8 = None
+        self._value = unistr
+        self._utf8 = None
 
-    def __repr__(w_self):
+    def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%r)" % (w_self.__class__.__name__, w_self._value)
+        return "%s(%r)" % (self.__class__.__name__, self._value)
 
-    def unwrap(w_self, space):
+    def unwrap(self, space):
         # for testing
-        return w_self._value
+        return self._value
 
-    def create_if_subclassed(w_self):
-        if type(w_self) is W_UnicodeObject:
-            return w_self
-        return W_UnicodeObject(w_self._value)
+    def create_if_subclassed(self):
+        if type(self) is W_UnicodeObject:
+            return self
+        return W_UnicodeObject(self._value)
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_UnicodeObject):
@@ -75,8 +75,8 @@
         self._utf8 = identifier
         return identifier
 
-    def listview_unicode(w_self):
-        return _create_list_from_unicode(w_self._value)
+    def listview_unicode(self):
+        return _create_list_from_unicode(self._value)
 
     def ord(self, space):
         if len(self._value) != 1:

From pypy.commits at gmail.com  Mon Jun 20 18:16:34 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge with upstream I hope.
Message-ID: <57686b42.2523c20a.e0232.ffffa36b@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85267:57b204adccaa
Date: 2016-05-22 13:06 -0400
http://bitbucket.org/pypy/pypy/changeset/57b204adccaa/

Log:	Merge with upstream I hope.

diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -4526,6 +4526,10 @@
 
         # make sure we have an example of each type of descriptor
         for d, n in zip(descriptors, types):
+            if (support.check_impl_detail(pypy=True) and
+                n in ('method', 'member', 'wrapper')):
+                # PyPy doesn't have these
+                continue
             self.assertEqual(type(d).__name__, n + '_descriptor')
 
         for d in descriptors:
@@ -4539,7 +4543,7 @@
 
         class X:
             pass
-        with self.assertRaises(TypeError):
+        with self.assertRaises((AttributeError, TypeError)):
             del X.__qualname__
 
         self.assertRaises(TypeError, type.__dict__['__qualname__'].__set__,
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -277,7 +277,16 @@
     if StdErrPrinter is not None:
         sys.stderr = sys.__stderr__ = StdErrPrinter(2)
 
-    if 1:  # keep indentation
+    # Hack to avoid recursion issues during bootstrapping: pre-import
+    # the utf-8 and latin-1 codecs
+    encerr = None
+    try:
+        import encodings.utf_8
+        import encodings.latin_1
+    except ImportError as e:
+        encerr = e
+
+    try:
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
         else:
@@ -296,6 +305,10 @@
             print("Python error:  is a directory, cannot continue",
                   file=sys.stderr)
             os._exit(1)
+    finally:
+        if encerr:
+            display_exception(encerr)
+            del encerr
 
 def create_stdio(fd, writing, name, encoding, errors, unbuffered):
     import io
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1138,12 +1138,14 @@
             old_last_exception = self.last_exception
             self.last_exception = operr
             w_traceback = self.space.wrap(operr.get_traceback())
-            w_suppress = self.call_contextmanager_exit_function(
-                w_exitfunc,
-                operr.w_type,
-                operr.get_w_value(self.space),
-                w_traceback)
-            self.last_exception = old_last_exception
+            try:
+                w_suppress = self.call_contextmanager_exit_function(
+                    w_exitfunc,
+                    operr.w_type,
+                    operr.get_w_value(self.space),
+                    w_traceback)
+            finally:
+                self.last_exception = old_last_exception
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.
                 self.settopvalue(self.space.w_None)
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -439,7 +439,6 @@
             fail('No exception raised')
 
     def test_context_with_suppressed(self):
-        # XXX: requires with statement's WHY_SILENCED
         class RaiseExc:
             def __init__(self, exc):
                 self.exc = exc
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -60,6 +60,7 @@
     def test_descr_getsetproperty(self):
         from types import FrameType
         assert FrameType.f_lineno.__name__ == 'f_lineno'
+        assert FrameType.f_lineno.__qualname__ == 'frame.f_lineno'
         assert FrameType.f_lineno.__objclass__ is FrameType
         class A(object):
             pass
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -263,6 +263,7 @@
         self.doc = doc
         self.reqcls = cls
         self.name = ''
+        self.qualname = None
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
 
@@ -313,6 +314,21 @@
                 self.reqcls, Arguments(space, [w_obj,
                                                space.wrap(self.name)]))
 
+    def descr_get_qualname(self, space):
+        if self.qualname is None:
+            self.qualname = self._calculate_qualname(space)
+        return self.qualname
+
+    def _calculate_qualname(self, space):
+        if self.reqcls is None:
+            type_qualname = u'?'
+        else:
+            w_type = space.gettypeobject(self.reqcls.typedef)
+            type_qualname = space.unicode_w(
+                space.getattr(w_type, space.wrap('__qualname__')))
+        qualname = u"%s.%s" % (type_qualname, self.name.decode('utf-8'))
+        return space.wrap(qualname)
+
     def descr_get_objclass(space, property):
         return property.objclass_getter(space)
 
@@ -351,6 +367,7 @@
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
     __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __qualname__ = GetSetProperty(GetSetProperty.descr_get_qualname),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
     __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
     )
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -72,8 +72,8 @@
         'max'           : 'functional.max',
         'reversed'      : 'functional.W_ReversedIterator',
         'super'         : 'descriptor.W_Super',
-        'staticmethod'  : 'descriptor.StaticMethod',
-        'classmethod'   : 'descriptor.ClassMethod',
+        'staticmethod'  : 'pypy.interpreter.function.StaticMethod',
+        'classmethod'   : 'pypy.interpreter.function.ClassMethod',
         'property'      : 'descriptor.W_Property',
 
         'globals'       : 'interp_inspect.globals',
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -1,31 +1,41 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.function import StaticMethod, ClassMethod
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import (
-    TypeDef, interp_attrproperty_w, generic_new_descr, GetSetProperty)
+    GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty_w)
 from pypy.objspace.descroperation import object_getattribute
 
 
 class W_Super(W_Root):
-    def __init__(self, space, w_starttype, w_objtype, w_self):
+
+    def __init__(self, space):
+        self.w_starttype = None
+        self.w_objtype = None
+        self.w_self = None
+
+    def descr_init(self, space, w_starttype=None, w_obj_or_type=None):
+        if space.is_none(w_starttype):
+            w_starttype, w_obj_or_type = _super_from_frame(space)
+        if space.is_none(w_obj_or_type):
+            w_type = None  # unbound super object
+            w_obj_or_type = space.w_None
+        else:
+            w_type = _supercheck(space, w_starttype, w_obj_or_type)
         self.w_starttype = w_starttype
-        self.w_objtype = w_objtype
-        self.w_self = w_self
+        self.w_objtype = w_type
+        self.w_self = w_obj_or_type
 
     def get(self, space, w_obj, w_type=None):
-        w = space.wrap
         if self.w_self is None or space.is_w(w_obj, space.w_None):
-            return w(self)
+            return self
         else:
             # if type(self) is W_Super:
             #     XXX write a fast path for this common case
-            w_selftype = space.type(w(self))
+            w_selftype = space.type(self)
             return space.call_function(w_selftype, self.w_starttype, w_obj)
 
-    @unwrap_spec(name=str)
-    def getattribute(self, space, name):
-        w = space.wrap
+    def getattribute(self, space, w_name):
+        name = space.str_w(w_name)
         # only use a special logic for bound super objects and not for
         # getting the __class__ of the super object itself.
         if self.w_objtype is not None and name != '__class__':
@@ -45,73 +55,68 @@
                 return space.get_and_call_function(w_get, w_value,
                                                    w_obj, self.w_objtype)
         # fallback to object.__getattribute__()
-        return space.call_function(object_getattribute(space),
-                                   w(self), w(name))
+        return space.call_function(object_getattribute(space), self, w_name)
 
-def descr_new_super(space, w_subtype, w_starttype=None, w_obj_or_type=None):
-    if space.is_none(w_starttype):
-        # Call super(), without args -- fill in from __class__
-        # and first local variable on the stack.
-        ec = space.getexecutioncontext()
-        frame = ec.gettopframe()
-        code = frame.pycode
-        if not code:
-            raise oefmt(space.w_RuntimeError, "super(): no code object")
-        if code.co_argcount == 0:
-            raise oefmt(space.w_RuntimeError, "super(): no arguments")
-        w_obj = frame.locals_cells_stack_w[0]
-        if not w_obj:
-            raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
-        index = 0
-        for name in code.co_freevars:
-            if name == "__class__":
-                break
-            index += 1
-        else:
-            raise oefmt(space.w_RuntimeError,
-                        "super(): __class__ cell not found")
-        # a kind of LOAD_DEREF
-        cell = frame._getcell(len(code.co_cellvars) + index)
-        try:
-            w_starttype = cell.get()
-        except ValueError:
-            raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
-        w_obj_or_type = w_obj
+def _super_from_frame(space):
+    """super() without args -- fill in from __class__ and first local
+    variable on the stack.
+    """
+    frame = space.getexecutioncontext().gettopframe()
+    code = frame.pycode
+    if not code:
+        raise oefmt(space.w_RuntimeError, "super(): no code object")
+    if code.co_argcount == 0:
+        raise oefmt(space.w_RuntimeError, "super(): no arguments")
+    w_obj = frame.locals_cells_stack_w[0]
+    if not w_obj:
+        raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
+    for index, name in enumerate(code.co_freevars):
+        if name == "__class__":
+            break
+    else:
+        raise oefmt(space.w_RuntimeError, "super(): __class__ cell not found")
+    # a kind of LOAD_DEREF
+    cell = frame._getcell(len(code.co_cellvars) + index)
+    try:
+        w_starttype = cell.get()
+    except ValueError:
+        raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
+    return w_starttype, w_obj
 
-    if space.is_none(w_obj_or_type):
-        w_type = None  # unbound super object
-        w_obj_or_type = space.w_None
-    else:
-        w_objtype = space.type(w_obj_or_type)
-        if space.is_true(space.issubtype(w_objtype, space.w_type)) and \
-            space.is_true(space.issubtype(w_obj_or_type, w_starttype)):
-            w_type = w_obj_or_type # special case for class methods
-        elif space.is_true(space.issubtype(w_objtype, w_starttype)):
-            w_type = w_objtype # normal case
-        else:
-            try:
-                w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
-            except OperationError as o:
-                if not o.match(space, space.w_AttributeError):
-                    raise
-                w_type = w_objtype
-            if not space.is_true(space.issubtype(w_type, w_starttype)):
-                raise oefmt(space.w_TypeError,
-                            "super(type, obj): obj must be an instance or "
-                            "subtype of type")
-    # XXX the details of how allocate_instance() should be used are not
-    # really well defined
-    w_result = space.allocate_instance(W_Super, w_subtype)
-    W_Super.__init__(w_result, space, w_starttype, w_type, w_obj_or_type)
-    return w_result
+def _supercheck(space, w_starttype, w_obj_or_type):
+    """Check that the super() call makes sense. Returns a type"""
+    w_objtype = space.type(w_obj_or_type)
+
+    if (space.is_true(space.issubtype(w_objtype, space.w_type)) and
+        space.is_true(space.issubtype(w_obj_or_type, w_starttype))):
+        # special case for class methods
+        return w_obj_or_type
+
+    if space.is_true(space.issubtype(w_objtype, w_starttype)):
+        # normal case
+        return w_objtype
+
+    try:
+        w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
+    except OperationError as e:
+        if not e.match(space, space.w_AttributeError):
+            raise
+        w_type = w_objtype
+
+    if space.is_true(space.issubtype(w_type, w_starttype)):
+        return w_type
+    raise oefmt(space.w_TypeError,
+                "super(type, obj): obj must be an instance or subtype of type")
 
 W_Super.typedef = TypeDef(
     'super',
-    __new__          = interp2app(descr_new_super),
+    __new__          = generic_new_descr(W_Super),
+    __init__         = interp2app(W_Super.descr_init),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
-    __doc__          =     """super(type) -> unbound super object
+    __doc__          =     """\
+super(type) -> unbound super object
 super(type, obj) -> bound super object; requires isinstance(obj, type)
 super(type, type2) -> bound super object; requires issubclass(type2, type)
 
@@ -129,10 +134,10 @@
     def __init__(self, space):
         pass
 
-    @unwrap_spec(w_fget = WrappedDefault(None),
-                 w_fset = WrappedDefault(None),
-                 w_fdel = WrappedDefault(None),
-                 w_doc = WrappedDefault(None))
+    @unwrap_spec(w_fget=WrappedDefault(None),
+                 w_fset=WrappedDefault(None),
+                 w_fdel=WrappedDefault(None),
+                 w_doc=WrappedDefault(None))
     def init(self, space, w_fget=None, w_fset=None, w_fdel=None, w_doc=None):
         self.w_fget = w_fget
         self.w_fset = w_fset
@@ -142,18 +147,17 @@
         # our __doc__ comes from the getter if we don't have an explicit one
         if (space.is_w(self.w_doc, space.w_None) and
             not space.is_w(self.w_fget, space.w_None)):
-            w_getter_doc = space.findattr(self.w_fget, space.wrap("__doc__"))
+            w_getter_doc = space.findattr(self.w_fget, space.wrap('__doc__'))
             if w_getter_doc is not None:
                 if type(self) is W_Property:
                     self.w_doc = w_getter_doc
                 else:
-                    space.setattr(space.wrap(self), space.wrap("__doc__"),
-                                  w_getter_doc)
+                    space.setattr(self, space.wrap('__doc__'), w_getter_doc)
                 self.getter_doc = True
 
     def get(self, space, w_obj, w_objtype=None):
         if space.is_w(w_obj, space.w_None):
-            return space.wrap(self)
+            return self
         if space.is_w(self.w_fget, space.w_None):
             raise oefmt(space.w_AttributeError, "unreadable attribute")
         return space.call_function(self.w_fget, w_obj)
@@ -191,7 +195,8 @@
         else:
             w_doc = self.w_doc
         w_type = self.getclass(space)
-        return space.call_function(w_type, w_getter, w_setter, w_deleter, w_doc)
+        return space.call_function(w_type, w_getter, w_setter, w_deleter,
+                                   w_doc)
 
     def descr_isabstract(self, space):
         return space.newbool(space.isabstractmethod_w(self.w_fget) or
@@ -200,7 +205,8 @@
 
 W_Property.typedef = TypeDef(
     'property',
-    __doc__ = '''property(fget=None, fset=None, fdel=None, doc=None) -> property attribute
+    __doc__ = '''\
+property(fget=None, fset=None, fdel=None, doc=None) -> property attribute
 
 fget is a function to be used for getting an attribute value, and likewise
 fset is a function for setting, and fdel a function for deleting, an
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -233,10 +233,9 @@
     # __________ app-level attributes __________
     def dir(self):
         space = self.space
-        w_self = space.wrap(self)
         lst = [space.wrap(name)
                   for name in _name_of_attributes
-                  if space.findattr(w_self, space.wrap(name)) is not None]
+                  if space.findattr(self, space.wrap(name)) is not None]
         return space.newlist(lst)
 
     def _fget(self, attrchar):
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -141,8 +141,6 @@
 
 class AppTestPartialEvaluation:
     spaceconfig = dict(usemodules=['array',])
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_winreg')
 
     def test_partial_utf8(self):
         import _codecs
@@ -767,7 +765,7 @@
         try:
             # test for non-latin1 codepage, more general test needed
             import winreg
-            key = winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE,
+            key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
                         r'System\CurrentControlSet\Control\Nls\CodePage')
             if winreg.QueryValueEx(key, 'ACP')[0] == u'1255':  # non-latin1
                 toencode = u'caf\xbf',b'caf\xbf'
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -389,20 +389,18 @@
     def copy(self):
         "Return a shallow copy of a deque."
         space = self.space
-        w_self = space.wrap(self)
         if self.maxlen == sys.maxint:
-            return space.call_function(space.type(w_self), w_self)
+            return space.call_function(space.type(self), self)
         else:
-            return space.call_function(space.type(w_self), w_self,
+            return space.call_function(space.type(self), self,
                                        space.wrap(self.maxlen))
 
     def reduce(self):
         "Return state information for pickling."
         space = self.space
-        w_self = space.wrap(self)
-        w_type = space.type(w_self)
-        w_dict = space.findattr(w_self, space.wrap('__dict__'))
-        w_list = space.call_function(space.w_list, w_self)
+        w_type = space.type(self)
+        w_dict = space.findattr(self, space.wrap('__dict__'))
+        w_list = space.call_function(space.w_list, self)
         if w_dict is None:
             if self.maxlen == sys.maxint:
                 result = [
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -156,12 +156,12 @@
 
 
 class W_WeakrefBase(W_Root):
-    def __init__(w_self, space, w_obj, w_callable):
+    def __init__(self, space, w_obj, w_callable):
         assert w_callable is not space.w_None    # should be really None
-        w_self.space = space
+        self.space = space
         assert w_obj is not None
-        w_self.w_obj_weak = weakref.ref(w_obj)
-        w_self.w_callable = w_callable
+        self.w_obj_weak = weakref.ref(w_obj)
+        self.w_callable = w_callable
 
     @jit.dont_look_inside
     def dereference(self):
@@ -171,8 +171,8 @@
     def clear(self):
         self.w_obj_weak = dead_ref
 
-    def activate_callback(w_self):
-        w_self.space.call_function(w_self.w_callable, w_self)
+    def activate_callback(self):
+        self.space.call_function(self.w_callable, self)
 
     def descr__repr__(self, space):
         w_obj = self.dereference()
@@ -189,9 +189,9 @@
 
 
 class W_Weakref(W_WeakrefBase):
-    def __init__(w_self, space, w_obj, w_callable):
-        W_WeakrefBase.__init__(w_self, space, w_obj, w_callable)
-        w_self.w_hash = None
+    def __init__(self, space, w_obj, w_callable):
+        W_WeakrefBase.__init__(self, space, w_obj, w_callable)
+        self.w_hash = None
 
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -19,7 +19,7 @@
     canSaveKey = True
 
 class AppTestHKey:
-    spaceconfig = dict(usemodules=('_winreg',))
+    #spaceconfig = dict(usemodules=('_winreg',))
 
     def test_repr(self):
         import winreg
@@ -27,7 +27,7 @@
         assert str(k) == ""
 
 class AppTestFfi:
-    spaceconfig = dict(usemodules=('_winreg',))
+    #spaceconfig = dict(usemodules=('_winreg',))
 
     def setup_class(cls):
         import _winreg
@@ -53,9 +53,9 @@
         w_test_data.append(w_btest)
 
     def teardown_class(cls):
-        import _winreg
+        import winreg
         try:
-            _winreg.DeleteKey(cls.root_key, cls.test_key_name)
+            winreg.DeleteKey(cls.root_key, cls.test_key_name)
         except WindowsError:
             pass
 
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -349,7 +349,7 @@
     compress = interp2app(W_BZ2Compressor.compress),
     flush = interp2app(W_BZ2Compressor.flush),
 )
-
+W_BZ2Compressor.typedef.acceptable_as_base_class = False
 
 def descr_decompressor__new__(space, w_subtype):
     x = space.allocate_instance(W_BZ2Decompressor, w_subtype)
@@ -457,3 +457,4 @@
     eof = GetSetProperty(W_BZ2Decompressor.eof_w),
     decompress = interp2app(W_BZ2Decompressor.decompress),
 )
+W_BZ2Decompressor.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -54,8 +54,6 @@
         st_flags = structseqfield(23, "user defined flags for file")
 
     def __init__(self, *args, **kw):
-        super(stat_result, self).__init__(*args, **kw)
-
         # If we have been initialized from a tuple,
         # st_?time might be set to None. Initialize it
         # from the int slots.
diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py
--- a/pypy/module/thread/os_local.py
+++ b/pypy/module/thread/os_local.py
@@ -50,10 +50,9 @@
         self.dicts[ec] = w_dict
         # call __init__
         try:
-            w_self = space.wrap(self)
-            w_type = space.type(w_self)
+            w_type = space.type(self)
             w_init = space.getattr(w_type, space.wrap("__init__"))
-            space.call_obj_args(w_init, w_self, self.initargs)
+            space.call_obj_args(w_init, self, self.initargs)
         except:
             # failed, forget w_dict and propagate the exception
             del self.dicts[ec]
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -364,6 +364,28 @@
         space = self.space
         return space.wrap_fsdecoded(self.filename)
 
+    def _find_loader(self, space, fullname):
+        filename = self.make_filename(fullname)
+        for _, _, ext in ENUMERATE_EXTS:
+            if self.have_modulefile(space, filename + ext):
+                return True, None
+        # See if this is a directory (part of a namespace pkg)
+        dirpath = self.prefix + fullname
+        if self.have_modulefile(space, dirpath + ZIPSEP):
+            return True, self.filename + os.path.sep + self.corr_zname(dirpath)
+        return False, None
+
+    @unwrap_spec(fullname='str0')
+    def find_loader(self, space, fullname, w_path=None):
+        found, ns_portion = self._find_loader(space, fullname)
+        if not found:
+            result = [space.w_None, space.newlist([])]
+        elif not ns_portion:
+            result = [self, space.newlist([])]
+        else:
+            result = [space.w_None, space.newlist([space.wrap(ns_portion)])]
+        return space.newtuple(result)
+
 def descr_new_zipimporter(space, w_type, w_name):
     name = space.fsencode_w(w_name)
     ok = False
@@ -422,6 +444,7 @@
     get_filename = interp2app(W_ZipImporter.get_filename),
     is_package  = interp2app(W_ZipImporter.is_package),
     load_module = interp2app(W_ZipImporter.load_module),
+    find_loader = interp2app(W_ZipImporter.find_loader),
     archive     = GetSetProperty(W_ZipImporter.getarchive),
     prefix      = GetSetProperty(W_ZipImporter.getprefix),
 )
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -440,6 +440,12 @@
         self.writefile('x1test/__init__.py', 'raise ValueError')
         raises(ValueError, __import__, 'x1test', None, None, [])
 
+    def test_namespace_pkg(self):
+        self.writefile('foo/', '')
+        self.writefile('foo/one.py', "attr = 'portion1 foo one'\n")
+        foo = __import__('foo.one', None, None, [])
+        assert foo.one.attr == 'portion1 foo one'
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):
diff --git a/pypy/objspace/std/noneobject.py b/pypy/objspace/std/noneobject.py
--- a/pypy/objspace/std/noneobject.py
+++ b/pypy/objspace/std/noneobject.py
@@ -4,7 +4,7 @@
 
 
 class W_NoneObject(W_Root):
-    def unwrap(w_self, space):
+    def unwrap(self, space):
         return None
 
     @staticmethod
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -84,23 +84,23 @@
     'object()' call."""
 
 
+def _excess_args(__args__):
+    return bool(__args__.arguments_w) or bool(__args__.keywords)
+
 def descr__new__(space, w_type, __args__):
-    from pypy.objspace.std.objectobject import W_ObjectObject
     from pypy.objspace.std.typeobject import _precheck_for_new
+    w_type = _precheck_for_new(space, w_type)
+
     # don't allow arguments if the default object.__init__() is about
     # to be called
-    w_type = _precheck_for_new(space, w_type)
-    w_parentinit, _ = w_type.lookup_where('__init__')
-    if w_parentinit is space.w_object:
-        try:
-            __args__.fixedunpack(0)
-        except ValueError:
+    if _excess_args(__args__):
+        w_parent_init, _ = space.lookup_in_type_where(w_type, '__init__')
+        if w_parent_init is space.w_object:
             raise oefmt(space.w_TypeError,
-                        "default __new__ takes no parameters")
+                        "object() takes no parameters")
     if w_type.is_abstract():
         _abstract_method_error(space, w_type)
-    w_obj = space.allocate_instance(W_ObjectObject, w_type)
-    return w_obj
+    return space.allocate_instance(W_ObjectObject, w_type)
 
 
 def descr___subclasshook__(space, __args__):
@@ -109,12 +109,10 @@
 
 def descr__init__(space, w_obj, __args__):
     # don't allow arguments unless __new__ is overridden
-    w_type = space.type(w_obj)
-    w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
-    if w_parent_new is space.w_object:
-        try:
-            __args__.fixedunpack(0)
-        except ValueError:
+    if _excess_args(__args__):
+        w_type = space.type(w_obj)
+        w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
+        if w_parent_new is space.w_object:
             raise oefmt(space.w_TypeError,
                         "object.__init__() takes no parameters")
 
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -12,13 +12,13 @@
 class W_SliceObject(W_Root):
     _immutable_fields_ = ['w_start', 'w_stop', 'w_step']
 
-    def __init__(w_self, w_start, w_stop, w_step):
+    def __init__(self, w_start, w_stop, w_step):
         assert w_start is not None
         assert w_stop is not None
         assert w_step is not None
-        w_self.w_start = w_start
-        w_self.w_stop = w_stop
-        w_self.w_step = w_step
+        self.w_start = w_start
+        self.w_stop = w_stop
+        self.w_step = w_step
 
     def unwrap(w_slice, space):
         return slice(space.unwrap(w_slice.w_start), space.unwrap(w_slice.w_stop), space.unwrap(w_slice.w_step))
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -26,10 +26,10 @@
         else:
             return self.w_str._value
 
-    def __repr__(w_self):
+    def __repr__(self):
         """ representation for debugging purposes """
         return "%s(%r[:%d])" % (
-            w_self.__class__.__name__, w_self.builder, w_self.length)
+            self.__class__.__name__, self.builder, self.length)
 
     def unwrap(self, space):
         return self.force()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -154,220 +154,220 @@
     w_new_function = None
 
     @dont_look_inside
-    def __init__(w_self, space, name, bases_w, dict_w,
+    def __init__(self, space, name, bases_w, dict_w,
                  overridetypedef=None, force_new_layout=False):
-        w_self.space = space
-        w_self.name = name
-        w_self.qualname = None
-        w_self.bases_w = bases_w
-        w_self.dict_w = dict_w
-        w_self.hasdict = False
-        w_self.hasuserdel = False
-        w_self.weakrefable = False
-        w_self.w_doc = space.w_None
-        w_self.weak_subclasses = []
-        w_self.flag_heaptype = False
-        w_self.flag_cpytype = False
-        w_self.flag_abstract = False
-        w_self.flag_sequence_bug_compat = False
-        w_self.flag_map_or_seq = '?'   # '?' means "don't know, check otherwise"
+        self.space = space
+        self.name = name
+        self.qualname = None
+        self.bases_w = bases_w
+        self.dict_w = dict_w
+        self.hasdict = False
+        self.hasuserdel = False
+        self.weakrefable = False
+        self.w_doc = space.w_None
+        self.weak_subclasses = []
+        self.flag_heaptype = False
+        self.flag_cpytype = False
+        self.flag_abstract = False
+        self.flag_sequence_bug_compat = False
+        self.flag_map_or_seq = '?'   # '?' means "don't know, check otherwise"
 
         if overridetypedef is not None:
             assert not force_new_layout
-            layout = setup_builtin_type(w_self, overridetypedef)
+            layout = setup_builtin_type(self, overridetypedef)
         else:
-            layout = setup_user_defined_type(w_self, force_new_layout)
-        w_self.layout = layout
+            layout = setup_user_defined_type(self, force_new_layout)
+        self.layout = layout
 
-        if not is_mro_purely_of_types(w_self.mro_w):
+        if not is_mro_purely_of_types(self.mro_w):
             pass
         else:
             # the _version_tag should change, whenever the content of
             # dict_w of any of the types in the mro changes, or if the mro
             # itself changes
-            w_self._version_tag = VersionTag()
+            self._version_tag = VersionTag()
         from pypy.objspace.std.mapdict import DictTerminator, NoDictTerminator
         # if the typedef has a dict, then the rpython-class does all the dict
         # management, which means from the point of view of mapdict there is no
         # dict.
-        typedef = w_self.layout.typedef
-        if (w_self.hasdict and not typedef.hasdict):
-            w_self.terminator = DictTerminator(space, w_self)
+        typedef = self.layout.typedef
+        if (self.hasdict and not typedef.hasdict):
+            self.terminator = DictTerminator(space, self)
         else:
-            w_self.terminator = NoDictTerminator(space, w_self)
+            self.terminator = NoDictTerminator(space, self)
 
     def __repr__(self):
         "NOT_RPYTHON"
         return '' % (self.name, id(self))
 
-    def mutated(w_self, key):
+    def mutated(self, key):
         """
         The type is being mutated. key is either the string containing the
         specific attribute which is being deleted/set or None to indicate a
         generic mutation.
         """
-        space = w_self.space
-        assert w_self.is_heaptype() or w_self.is_cpytype()
+        space = self.space
+        assert self.is_heaptype() or self.is_cpytype()
 
-        w_self.uses_object_getattribute = False
+        self.uses_object_getattribute = False
         # ^^^ conservative default, fixed during real usage
 
         if (key is None or key == '__eq__' or key == '__hash__'):
-            w_self.compares_by_identity_status = UNKNOWN
+            self.compares_by_identity_status = UNKNOWN
 
         if space.config.objspace.std.newshortcut:
-            w_self.w_new_function = None
+            self.w_new_function = None
 
-        if w_self._version_tag is not None:
-            w_self._version_tag = VersionTag()
+        if self._version_tag is not None:
+            self._version_tag = VersionTag()
 
-        subclasses_w = w_self.get_subclasses()
+        subclasses_w = self.get_subclasses()
         for w_subclass in subclasses_w:
             assert isinstance(w_subclass, W_TypeObject)
             w_subclass.mutated(key)
 
-    def version_tag(w_self):
-        if not we_are_jitted() or w_self.is_heaptype():
-            return w_self._version_tag
+    def version_tag(self):
+        if not we_are_jitted() or self.is_heaptype():
+            return self._version_tag
         # prebuilt objects cannot get their version_tag changed
-        return w_self._pure_version_tag()
+        return self._pure_version_tag()
 
     @elidable_promote()
-    def _pure_version_tag(w_self):
-        return w_self._version_tag
+    def _pure_version_tag(self):
+        return self._version_tag
 
-    def getattribute_if_not_from_object(w_self):
+    def getattribute_if_not_from_object(self):
         """ this method returns the applevel __getattribute__ if that is not
         the one from object, in which case it returns None """
         from pypy.objspace.descroperation import object_getattribute
         if not we_are_jitted():
-            if not w_self.uses_object_getattribute:
+            if not self.uses_object_getattribute:
                 # slow path: look for a custom __getattribute__ on the class
-                w_descr = w_self.lookup('__getattribute__')
+                w_descr = self.lookup('__getattribute__')
                 # if it was not actually overriden in the class, we remember this
                 # fact for the next time.
-                if w_descr is object_getattribute(w_self.space):
-                    w_self.uses_object_getattribute = True
+                if w_descr is object_getattribute(self.space):
+                    self.uses_object_getattribute = True
                 else:
                     return w_descr
             return None
         # in the JIT case, just use a lookup, because it is folded away
         # correctly using the version_tag
-        w_descr = w_self.lookup('__getattribute__')
-        if w_descr is not object_getattribute(w_self.space):
+        w_descr = self.lookup('__getattribute__')
+        if w_descr is not object_getattribute(self.space):
             return w_descr
 
-    def has_object_getattribute(w_self):
-        return w_self.getattribute_if_not_from_object() is None
+    def has_object_getattribute(self):
+        return self.getattribute_if_not_from_object() is None
 
-    def compares_by_identity(w_self):
+    def compares_by_identity(self):
         from pypy.objspace.descroperation import object_hash, type_eq
         #
-        if w_self.compares_by_identity_status != UNKNOWN:
+        if self.compares_by_identity_status != UNKNOWN:
             # fast path
-            return w_self.compares_by_identity_status == COMPARES_BY_IDENTITY
+            return self.compares_by_identity_status == COMPARES_BY_IDENTITY
         #
-        default_hash = object_hash(w_self.space)
-        my_eq = w_self.lookup('__eq__')
-        overrides_eq = (my_eq and my_eq is not type_eq(w_self.space))
+        default_hash = object_hash(self.space)
+        my_eq = self.lookup('__eq__')
+        overrides_eq = (my_eq and my_eq is not type_eq(self.space))
         overrides_eq_cmp_or_hash = (overrides_eq or
-                                    w_self.lookup('__hash__') is not default_hash)
+                                    self.lookup('__hash__') is not default_hash)
         if overrides_eq_cmp_or_hash:
-            w_self.compares_by_identity_status = OVERRIDES_EQ_CMP_OR_HASH
+            self.compares_by_identity_status = OVERRIDES_EQ_CMP_OR_HASH
         else:
-            w_self.compares_by_identity_status = COMPARES_BY_IDENTITY
-        return w_self.compares_by_identity_status == COMPARES_BY_IDENTITY
+            self.compares_by_identity_status = COMPARES_BY_IDENTITY
+        return self.compares_by_identity_status == COMPARES_BY_IDENTITY
 
-    def ready(w_self):
-        for w_base in w_self.bases_w:
+    def ready(self):
+        for w_base in self.bases_w:
             if not isinstance(w_base, W_TypeObject):
                 continue
-            w_base.add_subclass(w_self)
+            w_base.add_subclass(self)
 
     # compute a tuple that fully describes the instance layout
-    def get_full_instance_layout(w_self):
-        layout = w_self.layout
-        return (layout, w_self.hasdict, w_self.weakrefable)
+    def get_full_instance_layout(self):
+        layout = self.layout
+        return (layout, self.hasdict, self.weakrefable)
 
-    def compute_default_mro(w_self):
-        return compute_C3_mro(w_self.space, w_self)
+    def compute_default_mro(self):
+        return compute_C3_mro(self.space, self)
 
-    def getdictvalue(w_self, space, attr):
-        version_tag = w_self.version_tag()
+    def getdictvalue(self, space, attr):
+        version_tag = self.version_tag()
         if version_tag is not None:
             return unwrap_cell(
                 space,
-                w_self._pure_getdictvalue_no_unwrapping(
+                self._pure_getdictvalue_no_unwrapping(
                     space, version_tag, attr))
-        w_value = w_self._getdictvalue_no_unwrapping(space, attr)
+        w_value = self._getdictvalue_no_unwrapping(space, attr)
         return unwrap_cell(space, w_value)
 
-    def _getdictvalue_no_unwrapping(w_self, space, attr):
-        w_value = w_self.dict_w.get(attr, None)
-        if w_self.lazyloaders and w_value is None:
-            if attr in w_self.lazyloaders:
+    def _getdictvalue_no_unwrapping(self, space, attr):
+        w_value = self.dict_w.get(attr, None)
+        if self.lazyloaders and w_value is None:
+            if attr in self.lazyloaders:
                 # very clever next line: it forces the attr string
                 # to be interned.
                 space.new_interned_str(attr)
-                loader = w_self.lazyloaders[attr]
-                del w_self.lazyloaders[attr]
+                loader = self.lazyloaders[attr]
+                del self.lazyloaders[attr]
                 w_value = loader()
                 if w_value is not None:   # None means no such attribute
-                    w_self.dict_w[attr] = w_value
+                    self.dict_w[attr] = w_value
                     return w_value
         return w_value
 
     @elidable
-    def _pure_getdictvalue_no_unwrapping(w_self, space, version_tag, attr):
-        return w_self._getdictvalue_no_unwrapping(space, attr)
+    def _pure_getdictvalue_no_unwrapping(self, space, version_tag, attr):
+        return self._getdictvalue_no_unwrapping(space, attr)
 
-    def setdictvalue(w_self, space, name, w_value):
-        if not w_self.is_heaptype():
+    def setdictvalue(self, space, name, w_value):
+        if not self.is_heaptype():
             raise oefmt(space.w_TypeError,
-                        "can't set attributes on type object '%N'", w_self)
-        if name == "__del__" and name not in w_self.dict_w:
+                        "can't set attributes on type object '%N'", self)
+        if name == "__del__" and name not in self.dict_w:
             msg = ("a __del__ method added to an existing type will not be "
                    "called")
             space.warn(space.wrap(msg), space.w_RuntimeWarning)
-        version_tag = w_self.version_tag()
+        version_tag = self.version_tag()
         if version_tag is not None:
-            w_curr = w_self._pure_getdictvalue_no_unwrapping(
+            w_curr = self._pure_getdictvalue_no_unwrapping(
                     space, version_tag, name)
             w_value = write_cell(space, w_curr, w_value)
             if w_value is None:
                 return True
-        w_self.mutated(name)
-        w_self.dict_w[name] = w_value
+        self.mutated(name)
+        self.dict_w[name] = w_value
         return True
 
-    def deldictvalue(w_self, space, key):
-        if w_self.lazyloaders:
-            w_self._cleanup_()    # force un-lazification
-        if not w_self.is_heaptype():
+    def deldictvalue(self, space, key):
+        if self.lazyloaders:
+            self._cleanup_()    # force un-lazification
+        if not self.is_heaptype():
             raise oefmt(space.w_TypeError,
-                        "can't delete attributes on type object '%N'", w_self)
+                        "can't delete attributes on type object '%N'", self)
         try:
-            del w_self.dict_w[key]
+            del self.dict_w[key]
         except KeyError:
             return False
         else:
-            w_self.mutated(key)
+            self.mutated(key)
             return True
 
-    def lookup(w_self, name):
+    def lookup(self, name):
         # note that this doesn't call __get__ on the result at all
-        space = w_self.space
-        return w_self.lookup_where_with_method_cache(name)[1]
+        space = self.space
+        return self.lookup_where_with_method_cache(name)[1]
 
-    def lookup_where(w_self, name):
-        space = w_self.space
-        return w_self.lookup_where_with_method_cache(name)
+    def lookup_where(self, name):
+        space = self.space
+        return self.lookup_where_with_method_cache(name)
 
     @unroll_safe
-    def lookup_starting_at(w_self, w_starttype, name):
-        space = w_self.space
+    def lookup_starting_at(self, w_starttype, name):
+        space = self.space
         look = False
-        for w_class in w_self.mro_w:
+        for w_class in self.mro_w:
             if w_class is w_starttype:
                 look = True
             elif look:
@@ -377,54 +377,54 @@
         return None
 
     @unroll_safe
-    def _lookup(w_self, key):
+    def _lookup(self, key):
         # nowadays, only called from ../../tool/ann_override.py
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             w_value = w_class.getdictvalue(space, key)
             if w_value is not None:
                 return w_value
         return None
 
     @unroll_safe
-    def _lookup_where(w_self, key):
+    def _lookup_where(self, key):
         # like _lookup() but also returns the parent class in which the
         # attribute was found
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             w_value = w_class.getdictvalue(space, key)
             if w_value is not None:
                 return w_class, w_value
         return None, None
 
-    def _lookup_where_all_typeobjects(w_self, key):
-        # like _lookup_where(), but when we know that w_self.mro_w only
+    def _lookup_where_all_typeobjects(self, key):
+        # like _lookup_where(), but when we know that self.mro_w only
         # contains W_TypeObjects.  (It differs from _lookup_where() mostly
         # from a JIT point of view: it cannot invoke arbitrary Python code.)
-        space = w_self.space
-        for w_class in w_self.mro_w:
+        space = self.space
+        for w_class in self.mro_w:
             assert isinstance(w_class, W_TypeObject)
             w_value = w_class._getdictvalue_no_unwrapping(space, key)
             if w_value is not None:
                 return w_class, w_value
         return None, None
 
-    def lookup_where_with_method_cache(w_self, name):
-        space = w_self.space
-        promote(w_self)
-        version_tag = promote(w_self.version_tag())
+    def lookup_where_with_method_cache(self, name):
+        space = self.space
+        promote(self)
+        version_tag = promote(self.version_tag())
         if version_tag is None:
-            tup = w_self._lookup_where(name)
+            tup = self._lookup_where(name)
             return tup
-        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        tup_w = self._pure_lookup_where_with_method_cache(name, version_tag)
         w_class, w_value = tup_w
         if isinstance(w_value, MutableCell):
             return w_class, w_value.unwrap_cell(space)
         return tup_w   # don't make a new tuple, reuse the old one
 
     @elidable
-    def _pure_lookup_where_with_method_cache(w_self, name, version_tag):
-        space = w_self.space
+    def _pure_lookup_where_with_method_cache(self, name, version_tag):
+        space = self.space
         cache = space.fromcache(MethodCache)
         SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp
         SHIFT1 = SHIFT2 - 5
@@ -449,70 +449,70 @@
                 tup = cache.lookup_where[method_hash]
                 if space.config.objspace.std.withmethodcachecounter:
                     cache.hits[name] = cache.hits.get(name, 0) + 1
-#                print "hit", w_self, name
+#                print "hit", self, name
                 return tup
-        tup = w_self._lookup_where_all_typeobjects(name)
+        tup = self._lookup_where_all_typeobjects(name)
         cache.versions[method_hash] = version_tag
         cache.names[method_hash] = name
         cache.lookup_where[method_hash] = tup
         if space.config.objspace.std.withmethodcachecounter:
             cache.misses[name] = cache.misses.get(name, 0) + 1
-#        print "miss", w_self, name
+#        print "miss", self, name
         return tup
 
-    def check_user_subclass(w_self, w_subtype):
-        space = w_self.space
+    def check_user_subclass(self, w_subtype):
+        space = self.space
         if not isinstance(w_subtype, W_TypeObject):
             raise oefmt(space.w_TypeError,
                         "X is not a type object ('%T')", w_subtype)
-        if not w_subtype.issubtype(w_self):
+        if not w_subtype.issubtype(self):
             raise oefmt(space.w_TypeError,
                         "%N.__new__(%N): %N is not a subtype of %N",
-                        w_self, w_subtype, w_subtype, w_self)
-        if w_self.layout.typedef is not w_subtype.layout.typedef:
+                        self, w_subtype, w_subtype, self)
+        if self.layout.typedef is not w_subtype.layout.typedef:
             raise oefmt(space.w_TypeError,
                         "%N.__new__(%N) is not safe, use %N.__new__()",
-                        w_self, w_subtype, w_subtype)
+                        self, w_subtype, w_subtype)
         return w_subtype
 
-    def _cleanup_(w_self):
+    def _cleanup_(self):
         "NOT_RPYTHON.  Forces the lazy attributes to be computed."
-        if 'lazyloaders' in w_self.__dict__:
-            for attr in w_self.lazyloaders.keys():
-                w_self.getdictvalue(w_self.space, attr)
-            del w_self.lazyloaders
+        if 'lazyloaders' in self.__dict__:
+            for attr in self.lazyloaders.keys():
+                self.getdictvalue(self.space, attr)
+            del self.lazyloaders
 
-    def getdict(w_self, space): # returning a dict-proxy!
+    def getdict(self, space): # returning a dict-proxy!
         from pypy.objspace.std.dictproxyobject import DictProxyStrategy
         from pypy.objspace.std.dictproxyobject import W_DictProxyObject
-        if w_self.lazyloaders:
-            w_self._cleanup_()    # force un-lazification
+        if self.lazyloaders:
+            self._cleanup_()    # force un-lazification
         strategy = space.fromcache(DictProxyStrategy)
-        storage = strategy.erase(w_self)
+        storage = strategy.erase(self)
         return W_DictProxyObject(space, strategy, storage)
 
-    def is_heaptype(w_self):
-        return w_self.flag_heaptype
+    def is_heaptype(self):
+        return self.flag_heaptype
 
-    def is_cpytype(w_self):
-        return w_self.flag_cpytype
+    def is_cpytype(self):
+        return self.flag_cpytype
 
-    def is_abstract(w_self):
-        return w_self.flag_abstract
+    def is_abstract(self):
+        return self.flag_abstract
 
-    def set_abstract(w_self, abstract):
-        w_self.flag_abstract = bool(abstract)
+    def set_abstract(self, abstract):
+        self.flag_abstract = bool(abstract)
 
-    def issubtype(w_self, w_type):
-        promote(w_self)
+    def issubtype(self, w_type):
+        promote(self)
         promote(w_type)
         if we_are_jitted():
-            version_tag1 = w_self.version_tag()
+            version_tag1 = self.version_tag()
             version_tag2 = w_type.version_tag()
             if version_tag1 is not None and version_tag2 is not None:
-                res = _pure_issubtype(w_self, w_type, version_tag1, version_tag2)
+                res = _pure_issubtype(self, w_type, version_tag1, version_tag2)
                 return res
-        return _issubtype(w_self, w_type)
+        return _issubtype(self, w_type)
 
     def get_module(self):
         space = self.space
@@ -540,8 +540,8 @@
     def getqualname(self, space):
         return self.qualname or self.getname(space)
 
-    def add_subclass(w_self, w_subclass):
-        space = w_self.space
+    def add_subclass(self, w_subclass):
+        space = self.space
         if not space.config.translation.rweakref:
             # We don't have weakrefs!  In this case, every class stores
             # subclasses in a non-weak list.  ALL CLASSES LEAK!  To make
@@ -554,26 +554,26 @@
 
         assert isinstance(w_subclass, W_TypeObject)
         newref = weakref.ref(w_subclass)
-        for i in range(len(w_self.weak_subclasses)):
-            ref = w_self.weak_subclasses[i]
+        for i in range(len(self.weak_subclasses)):
+            ref = self.weak_subclasses[i]
             if ref() is None:
-                w_self.weak_subclasses[i] = newref
+                self.weak_subclasses[i] = newref
                 return
         else:
-            w_self.weak_subclasses.append(newref)
+            self.weak_subclasses.append(newref)
 
-    def remove_subclass(w_self, w_subclass):
-        space = w_self.space
-        for i in range(len(w_self.weak_subclasses)):
-            ref = w_self.weak_subclasses[i]
+    def remove_subclass(self, w_subclass):
+        space = self.space
+        for i in range(len(self.weak_subclasses)):
+            ref = self.weak_subclasses[i]
             if ref() is w_subclass:
-                del w_self.weak_subclasses[i]
+                del self.weak_subclasses[i]
                 return
 
-    def get_subclasses(w_self):
-        space = w_self.space
+    def get_subclasses(self):
+        space = self.space
         subclasses_w = []
-        for ref in w_self.weak_subclasses:
+        for ref in self.weak_subclasses:
             w_ob = ref()
             if w_ob is not None:
                 subclasses_w.append(w_ob)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -26,23 +26,23 @@
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value', '_utf8?']
 
-    def __init__(w_self, unistr):
+    def __init__(self, unistr):
         assert isinstance(unistr, unicode)
-        w_self._value = unistr
-        w_self._utf8 = None
+        self._value = unistr
+        self._utf8 = None
 
-    def __repr__(w_self):
+    def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%r)" % (w_self.__class__.__name__, w_self._value)
+        return "%s(%r)" % (self.__class__.__name__, self._value)
 
-    def unwrap(w_self, space):
+    def unwrap(self, space):
         # for testing
-        return w_self._value
+        return self._value
 
-    def create_if_subclassed(w_self):
-        if type(w_self) is W_UnicodeObject:
-            return w_self
-        return W_UnicodeObject(w_self._value)
+    def create_if_subclassed(self):
+        if type(self) is W_UnicodeObject:
+            return self
+        return W_UnicodeObject(self._value)
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_UnicodeObject):
@@ -75,8 +75,8 @@
         self._utf8 = identifier
         return identifier
 
-    def listview_unicode(w_self):
-        return _create_list_from_unicode(w_self._value)
+    def listview_unicode(self):
+        return _create_list_from_unicode(self._value)
 
     def ord(self, space):
         if len(self._value) != 1:

From pypy.commits at gmail.com  Mon Jun 20 18:16:41 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge with upstream.
Message-ID: <57686b49.8a40c20a.3bfce.23c0@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85271:96a0f86b3857
Date: 2016-05-23 16:59 -0400
http://bitbucket.org/pypy/pypy/changeset/96a0f86b3857/

Log:	Merge with upstream.

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -417,7 +417,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True),
+    RegrTest('test_time.py', core=True, usemodules="struct"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -40,6 +40,7 @@
         'struct_time': 'app_time.struct_time',
         '__doc__': 'app_time.__doc__',
         'strptime': 'app_time.strptime',
+        'get_clock_info': 'app_time.get_clock_info'
     }
 
     def startup(self, space):
diff --git a/pypy/module/time/app_time.py b/pypy/module/time/app_time.py
--- a/pypy/module/time/app_time.py
+++ b/pypy/module/time/app_time.py
@@ -1,7 +1,8 @@
 # NOT_RPYTHON
 
 from _structseq import structseqtype, structseqfield
-
+from types import SimpleNamespace
+import time
 class struct_time(metaclass=structseqtype):
     __module__ = 'time'
     name = 'time.struct_time'
@@ -26,6 +27,28 @@
     import _strptime     # from the CPython standard library
     return _strptime._strptime_time(string, format)
 
+def get_clock_info(name):
+    info = SimpleNamespace()
+    info.implementation = ""
+    info.monotonic = 0
+    info.adjustable = 0
+    info.resolution = 1.0
+    print(id(info), "id in app")
+
+    if name == "time":
+        time.time(info)
+    elif name == "monotonic":
+        time.monotonic(info)
+    elif name == "clock":
+        time.clock(info)
+    elif name == "perf_counter":
+        time.perf_counter(info)
+    elif name == "process_time":
+        time.process_time(info)
+    else:
+        raise ValueError("unknown clock")
+    return info
+
 __doc__ = """This module provides various functions to manipulate time values.
 
 There are two standard representations of time.  One is the number
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -103,6 +103,14 @@
         def get_interrupt_event(self):
             return globalState.interrupt_event
 
+    # Can I just use one of the state classes above?
+    # I don't really get why an instance is better than a plain module
+    # attr, but following advice from armin
+    class TimeState(object):
+        def __init__(self):
+            self.n_overflow = 0
+            self.last_ticks = 0
+    time_state = TimeState()
 
 _includes = ["time.h"]
 if _POSIX:
@@ -118,6 +126,7 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
+    has_gettickcount64 = platform.Has("GetTickCount64")
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
@@ -185,6 +194,7 @@
 
 CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
 HAS_CLOCK_GETTIME = cConfig.has_clock_gettime
+HAS_GETTICKCOUNT64 = cConfig.has_gettickcount64
 clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
@@ -503,18 +513,19 @@
 
     Return the current time in seconds since the Epoch.
     Fractions of a second may be present if the system clock provides them."""
-
     secs = pytime.time()
     return space.wrap(secs)
 
-def clock(space):
-    """clock() -> floating point number
-
-    Return the CPU time or real time since the start of the process or since
-    the first call to clock().  This has as much precision as the system
-    records."""
-
-    return space.wrap(pytime.clock())
+# TODO: Remember what this is for...
+def get_time_time_clock_info(space, w_info):
+    # Can't piggy back on time.time because time.time delegates to the 
+    # host python's time.time (so we can't see the internals)
+    if HAS_CLOCK_GETTIME:
+        try:
+            res = clock_getres(space, cConfig.CLOCK_REALTIME)
+        except OperationError:
+            res = 1e-9
+    #else: ???
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -717,9 +728,55 @@
 if _WIN:
     # untested so far
     _GetTickCount64 = rwin32.winexternal('GetTickCount64', [], rffi.ULONGLONG)
+    _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
+    LPDWORD = rwin32.LPDWORD
+    _GetSystemTimeAdjustment = rwin32.winexternal(
+                                            'GetSystemTimeAdjustment',
+                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
+                                            rffi.INT)
 
-    def monotonic(space):
-        return space.wrap(_GetTickCount64() * 1e-3)
+    def monotonic(space, w_info=None):
+        result = 0
+        if HAS_GETTICKCOUNT64:
+            print('has count64'.encode('ascii')) 
+            result = _GetTickCount64() * 1e-3
+        else:
+            print("nocount64")
+            ticks = _GetTickCount()
+            if ticks < time_state.last_ticks:
+                time_state.n_overflow += 1
+            time_state.last_ticks = ticks
+            result = math.ldexp(time_state.n_overflow, 32)
+            result = result + ticks
+            result = result * 1e-3
+            
+        if w_info is not None:
+            if HAS_GETTICKCOUNT64:
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("GetTickCount64()"))
+            else:
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("GetTickCount()"))
+            resolution = 1e-7
+            print("creating a thing".encode("ascii"))
+            with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_adjustment, \
+                 lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_increment, \
+                 lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
+                print("CREATED".encode("ascii"))
+                ok = _GetSystemTimeAdjustment(time_adjustment,
+                                              time_increment,
+                                              is_time_adjustment_disabled)
+                if not ok:
+                    # Is this right? Cargo culting...
+                    raise wrap_windowserror(space,
+                        rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
+                resolution = resolution * time_increment[0]
+            print("out of with".encode("ascii"))
+            space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+            space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+            space.setattr(w_info, space.wrap("resolution"),
+                          space.wrap(resolution))
+        return space.wrap(result)
 
 elif _MACOSX:
     c_mach_timebase_info = external('mach_timebase_info',
@@ -730,13 +787,23 @@
     timebase_info = lltype.malloc(cConfig.TIMEBASE_INFO, flavor='raw',
                                   zero=True, immortal=True)
 
-    def monotonic(space):
+    def monotonic(space, w_info=None):
         if rffi.getintfield(timebase_info, 'c_denom') == 0:
             c_mach_timebase_info(timebase_info)
         time = rffi.cast(lltype.Signed, c_mach_absolute_time())
         numer = rffi.getintfield(timebase_info, 'c_numer')
         denom = rffi.getintfield(timebase_info, 'c_denom')
         nanosecs = time * numer / denom
+        if w_info is not None:
+            space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+            space.setattr(w_info, space.wrap("implementation"),
+                          space.wrap("mach_absolute_time()"))
+            space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+            space.setattr(w_info, space.wrap("resolution"),
+                          #Do I need to convert to float indside the division?
+                          # Looking at the C, I would say yes, but nanosecs
+                          # doesn't...
+                          space.wrap((numer / denom) * 1e-9))
         secs = nanosecs / 10**9
         rest = nanosecs % 10**9
         return space.wrap(float(secs) + float(rest) * 1e-9)
@@ -744,21 +811,49 @@
 else:
     assert _POSIX
     if cConfig.CLOCK_HIGHRES is not None:
-        def monotonic(space):
+        def monotonic(space, w_info=None):
+            if w_info is not None:
+                space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("clock_gettime(CLOCK_HIGHRES)"))
+                space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+                try:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(clock_getres(space, cConfig.CLOCK_HIGHRES)))
+                except OSError:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(1e-9))
+                
             return clock_gettime(space, cConfig.CLOCK_HIGHRES)
     else:
-        def monotonic(space):
+        def monotonic(space, w_info=None):
+            if w_info is not None:
+                space.setattr(w_info, space.wrap("monotonic"), space.w_True)
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("clock_gettime(CLOCK_MONOTONIC)"))
+                space.setattr(w_info, space.wrap("adjustable"), space.w_False)
+                try:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(clock_getres(space, cConfig.CLOCK_MONOTONIC)))
+                except OSError:
+                    space.setattr(w_info, space.wrap("resolution"),
+                                  space.wrap(1e-9))
+
             return clock_gettime(space, cConfig.CLOCK_MONOTONIC)
 
+if _WIN:
+    def perf_counter(space, w_info=None):
+        # What if the windows perf counter fails?
+        # Cpython falls back to monotonic and then clock
+        # Shouldn't we?
+        # TODO: Discuss on irc
 
-if _WIN:
-    def perf_counter(space):
+        # TODO: Figure out how to get at the internals of this
         return space.wrap(win_perf_counter())
 
 else:
-    def perf_counter(space):
-        return monotonic(space)
-
+    def perf_counter(space, w_info=None):
+        return monotonic(space, w_info=w_info)
 
 if _WIN:
     # untested so far
@@ -810,3 +905,54 @@
                     cpu_time = float(tms.c_tms_utime + tms.c_tms_stime)
                     return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
+
+if _WIN:
+    def clock(space, w_info=None):
+        """clock() -> floating point number
+
+        Return the CPU time or real time since the start of the process or since
+        the first call to clock().  This has as much precision as the system
+        records."""
+        return space.wrap(win_perf_counter(space, w_info=w_info))
+
+else:
+    _clock = external('clock', [], clock_t)
+    def clock(space, w_info=None):
+        """clock() -> floating point number
+
+        Return the CPU time or real time since the start of the process or since
+        the first call to clock().  This has as much precision as the system
+        records."""
+        value = _clock()
+        #Is this casting correct?
+        if value == rffi.cast(clock_t, -1):
+            raise RunTimeError("the processor time used is not available "
+                               "or its value cannot be represented")
+
+        print(w_info, "INFO")
+        if w_info is not None:
+            space.setattr(w_info, space.wrap("implementation"),
+                          space.wrap("clock()"))
+            space.setattr(w_info, space.wrap("resolution"),
+                          space.wrap(1.0 / CLOCKS_PER_SEC))
+            space.setattr(w_info, space.wrap("monotonic"),
+                          space.w_True)
+            space.setattr(w_info, space.wrap("adjustable"),
+                          space.w_False)
+        return space.wrap((1.0 * value) / CLOCKS_PER_SEC)
+
+
+def get_clock_info_dict(space, name):
+    if name == "time":
+        return 5#floattime(info)
+    elif name == "monotonic":
+        return monotonic(info)
+    elif name == "clock":
+        return clock(info)
+    elif name == "perf_counter":
+        return perf_counter(info)
+    elif name == "process_time":
+        return 5#process_time(info)
+    else:
+        raise ValueError("unknown clock")
+
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -379,3 +379,21 @@
         t2 = time.process_time()
         # process_time() should not include time spent during sleep
         assert (t2 - t1) < 0.05
+
+    def test_get_clock_info_monotonic(self):
+        import time
+        clock_info = time.get_clock_info("monotonic")
+        assert clock_info.monotonic
+        assert not clock_info.adjustable
+        # Not really sure what to test about this
+        # At least this tests that the attr exists...
+        assert clock_info.resolution > 0
+
+    def test_get_clock_info_clock(self):
+        import time
+        clock_info = time.get_clock_info("clock")
+        assert clock_info.monotonic
+        assert not clock_info.adjustable
+        # Not really sure what to test about this
+        # At least this tests that the attr exists...
+        assert clock_info.resolution > 0
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -46,6 +46,7 @@
         LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP)
         LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP)
         LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.UINTP)
+        LPBOOL = rffi_platform.SimpleType("LPBOOL", rffi.LONGP)
         SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T)
         ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG)
 

From pypy.commits at gmail.com  Mon Jun 20 18:16:39 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fixes and useless logs. Will remove useles
 logs later.
Message-ID: <57686b47.8438c20a.7991b.1059@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85270:0dcaec29c001
Date: 2016-05-22 18:43 -0400
http://bitbucket.org/pypy/pypy/changeset/0dcaec29c001/

Log:	Fixes and useless logs. Will remove useles logs later.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -732,14 +732,16 @@
     LPDWORD = rwin32.LPDWORD
     _GetSystemTimeAdjustment = rwin32.winexternal(
                                             'GetSystemTimeAdjustment',
-                                            [LPDWORD, LPDWORD, rffi.LPBOOL], 
+                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
                                             rffi.INT)
 
     def monotonic(space, w_info=None):
         result = 0
         if HAS_GETTICKCOUNT64:
+            print('has count64'.encode('ascii')) 
             result = _GetTickCount64() * 1e-3
         else:
+            print("nocount64")
             ticks = _GetTickCount()
             if ticks < time_state.last_ticks:
                 time_state.n_overflow += 1
@@ -756,9 +758,11 @@
                 space.setattr(w_info, space.wrap("implementation"),
                               space.wrap("GetTickCount()"))
             resolution = 1e-7
-            with lltype.scoped_alloc(rwin32.LPDWORD) as time_adjustment, \
-                 lltype.scoped_alloc(rwin32.LPDWORD) as time_increment, \
-                 lltype.scoped_alloc(rwin32.LPBOOL) as is_time_adjustment_disabled:
+            print("creating a thing".encode("ascii"))
+            with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_adjustment, \
+                 lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_increment, \
+                 lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
+                print("CREATED".encode("ascii"))
                 ok = _GetSystemTimeAdjustment(time_adjustment,
                                               time_increment,
                                               is_time_adjustment_disabled)
@@ -766,8 +770,8 @@
                     # Is this right? Cargo culting...
                     raise wrap_windowserror(space,
                         rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
-                resolution = resolution * time_increment
-                
+                resolution = resolution * time_increment[0]
+            print("out of with".encode("ascii"))
             space.setattr(w_info, space.wrap("monotonic"), space.w_True)
             space.setattr(w_info, space.wrap("adjustable"), space.w_False)
             space.setattr(w_info, space.wrap("resolution"),

From pypy.commits at gmail.com  Mon Jun 20 18:16:48 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Implement dynamic lookup properly.
Message-ID: <57686b50.4aa71c0a.c06a3.355c@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85275:c16046ea79d3
Date: 2016-05-24 21:30 -0400
http://bitbucket.org/pypy/pypy/changeset/c16046ea79d3/

Log:	Implement dynamic lookup properly.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -111,13 +111,6 @@
             self.n_overflow = 0
             self.last_ticks = 0
     time_state = TimeState()
-    from rpython.rlib.rdynload import GetModuleHandle, dlsym
-    hKernel32 = GetModuleHandle("KERNEL32")
-    try:
-        dlsym(hKernel32, 'GetFinalPathNameByHandleW')
-        HAS_GETTICKCOUNT64 = True
-    except KeyError:
-        HAS_GETTICKCOUNT64 = False
 
 _includes = ["time.h"]
 if _POSIX:
@@ -733,13 +726,19 @@
 
 if _WIN:
     # untested so far
-    _GetTickCount64 = rwin32.winexternal('GetTickCount64', [], rffi.ULONGLONG)
     _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
     LPDWORD = rwin32.LPDWORD
     _GetSystemTimeAdjustment = rwin32.winexternal(
                                             'GetSystemTimeAdjustment',
                                             [LPDWORD, LPDWORD, rwin32.LPBOOL], 
                                             rffi.INT)
+    from rpython.rlib.rdynload import GetModuleHandle, dlsym
+    hKernel32 = GetModuleHandle("KERNEL32")
+    try:
+        _GetTickCount64 = dlsym(hKernel32, 'GetTickCount64')
+        HAS_GETTICKCOUNT64 = True
+    except KeyError:
+        HAS_GETTICKCOUNT64 = False
 
     def monotonic(space, w_info=None):
         result = 0

From pypy.commits at gmail.com  Mon Jun 20 18:16:53 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Committing so I can diff between mine and
 upstream.
Message-ID: <57686b55.46c21c0a.779c4.ffffc82b@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85278:aa738befc92c
Date: 2016-05-29 11:16 -0400
http://bitbucket.org/pypy/pypy/changeset/aa738befc92c/

Log:	Committing so I can diff between mine and upstream.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -182,11 +182,12 @@
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
     # TODO: Figure out how to implement this...
+    u = platform.Struct("struct u", [("LowPart", rwin32.DWORD),
+        ("HighPart", rwin32.DWORD)])
+
     CConfig.ULARGE_INTEGER = platform.Struct("struct ULARGE_INTEGER", [
-        ("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+        ("LowPart", rwin32.DWORD), ("HighPart", rwin32.DWORD),
+        ("u", u), ("QuadPart", rffi.ULONGLONG)])
 
 if _MACOSX:
     CConfig.TIMEBASE_INFO = platform.Struct("struct mach_timebase_info", [
@@ -234,17 +235,38 @@
 rffi.VOIDP], 
                               rffi.INT)
     if _WIN:
-       GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime', 
+        GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime', 
                                           [rwin32.FILETIME], 
                                           lltype.VOID)
+        LPDWORD = rwin32.LPDWORD
+        _GetSystemTimeAdjustment = rwin32.winexternal(
+                                                'GetSystemTimeAdjustment',
+                                                [LPDWORD, LPDWORD, rwin32.LPBOOL], 
+                                                rffi.INT)
         def gettimeofday(space, w_info=None):
-            with lltype.scoped_alloc(rwin32.FILETIME) as system_time, 
+            with lltype.scoped_alloc(rwin32.FILETIME) as system_time, \
+                 lltype.scoped_alloc(CConfig.ULARGE_INTEGER) as large, \
+                 lltype.scoped_alloc(rffi.ULONGLONG) as microseconds:
+
                 GetSystemTimeAsFileTime(system_time)
-                
+                large.u.LowPart = system_time.dwLowDateTime
+                large.u.HighPart = system_time.dwHighDateTime
+                microseconds = system_time.Quadpart / (10 - 11644473600000000)
+                tp.tv_sec = microseconds / 1000000
+                tp.tv_usec = microseconds % 1000000
+                if w_info:
+                    with lltype.scoped_alloc(rwin32.DWORD) as time_adjustment, \
+                         lltype.scoped_alloc(rwin32.DWORD) as time_increment, \
+                         lltype.scoped_alloc(rwin32.BOOL) as is_time_adjustmentDisabled:
+                        w_info.implementation = "GetSystemTimeAsFileTime()"
+                        w_info.monotonic = space.w_False
+                        _GetSystemTimeAdjustment(time_adjustment, time_increment,
+                                                is_time_adjustment_disabled)
+                        w_info.resolution = time_increment * 1e-7
+                        w_info.adjustable = space.w_True
+            # TODO: Find docs for ftime
+            return space.wrap(1)
 
-                seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
-
-            return space.wrap(seconds)
     else:
         def gettimeofday(space, w_info=None):
             with lltype.scoped_alloc(CConfig.timeval) as timeval:
@@ -581,7 +603,7 @@
 def get_time_time_clock_info(space, w_info):
     # Can't piggy back on time.time because time.time delegates to the 
     # host python's time.time (so we can't see the internals)
-    if HAS_CLOCK_GETTIME:
+    if HAS_CLOCK_GETTIME and False:
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(cConfig.CLOCK_REALTIME, timespec)
             if ret != 0:
@@ -804,11 +826,6 @@
 if _WIN:
     # untested so far
     _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
-    LPDWORD = rwin32.LPDWORD
-    _GetSystemTimeAdjustment = rwin32.winexternal(
-                                            'GetSystemTimeAdjustment',
-                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
-                                            rffi.INT)
     def monotonic(space, w_info=None):
         result = 0
         if HAS_GETTICKCOUNT64:

From pypy.commits at gmail.com  Mon Jun 20 18:16:46 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Let's try this.
Message-ID: <57686b4e.46c21c0a.779c4.ffffc827@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85274:4c460de3aa2a
Date: 2016-05-24 21:16 -0400
http://bitbucket.org/pypy/pypy/changeset/4c460de3aa2a/

Log:	Let's try this.

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -42,6 +42,16 @@
         'scripts': '{base}/bin',
         'data': '{base}',
         },
+    'pypy': {
+        'stdlib': '{installed_base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{installed_base}/include',
+        'platinclude': '{installed_base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{installed_base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -198,7 +208,9 @@
 
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
diff --git a/lib-python/3/tempfile.py b/lib-python/3/tempfile.py
--- a/lib-python/3/tempfile.py
+++ b/lib-python/3/tempfile.py
@@ -34,6 +34,7 @@
 import os as _os
 import shutil as _shutil
 import errno as _errno
+import weakref as _weakref
 from random import Random as _Random
 
 try:
@@ -686,6 +687,7 @@
 
     def __init__(self, suffix="", prefix=template, dir=None):
         self.name = mkdtemp(suffix, prefix, dir)
+        _tmpdirs.add(self)
 
     def __repr__(self):
         return "<{} {!r}>".format(self.__class__.__name__, self.name)
@@ -714,6 +716,7 @@
 
     def __exit__(self, exc, value, tb):
         self.cleanup()
+        _tmpdirs.discard(self)
 
     def __del__(self):
         # Issue a ResourceWarning if implicit cleanup needed
@@ -736,10 +739,23 @@
         except _OSError:
             pass
 
+_tmpdirs = _weakref.WeakSet()
 _is_running = True
 
+def _tmpdir_cleanup():
+    while _tmpdirs:
+        try:
+            tmpdir = _tmpdirs.pop()
+        except KeyError:
+            break
+        try:
+            tmpdir.cleanup(_warn=True)
+        except:
+            pass
+
 def _on_shutdown():
     global _is_running
+    _tmpdir_cleanup()
     _is_running = False
 
 _atexit.register(_on_shutdown)
diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -4533,6 +4533,9 @@
             self.assertEqual(type(d).__name__, n + '_descriptor')
 
         for d in descriptors:
+            if (support.check_impl_detail(pypy=True) and
+                not hasattr(d, '__objclass__')):
+                continue
             qualname = d.__objclass__.__qualname__ + '.' + d.__name__
             self.assertEqual(d.__qualname__, qualname)
 
@@ -4574,6 +4577,8 @@
         for o in gc.get_objects():
             self.assertIsNot(type(o), X)
 
+    @unittest.skipIf(support.check_impl_detail(pypy=True),
+                     "https://bitbucket.org/pypy/pypy/issues/2306")
     def test_object_new_and_init_with_parameters(self):
         # See issue #1683368
         class OverrideNeither:
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -239,7 +239,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     @skip_unless_symlink
@@ -345,6 +345,7 @@
             self.assertEqual(status, 0)
             self.assertEqual(my_platform, test_platform)
 
+    @impl_detail("Test is not PyPy compatible", pypy=False)
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')
@@ -379,7 +380,7 @@
 
 class MakefileTests(unittest.TestCase):
 
-    @impl_detail("PyPy lacks sysconfig.get_makefile_filename", pypy=False)
+    @impl_detail("Test is not PyPy compatible", pypy=False)
     @unittest.skipIf(sys.platform.startswith('win'),
                      'Test is not Windows compatible')
     def test_get_makefile_filename(self):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -18,6 +18,8 @@
     "exceptions", "_io", "sys", "builtins", "posix", "_warnings",
     "itertools", "_frozen_importlib",
 ])
+if sys.platform == "win32":
+    essential_modules.add("_winreg")
 
 default_modules = essential_modules.copy()
 default_modules.update([
@@ -60,7 +62,6 @@
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
-    working_modules.add("_winreg")
     # unix only modules
     for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses",
                  "_posixsubprocess"]:
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -185,29 +185,7 @@
     __multicall__.execute()
 
 def pytest_runtest_teardown(__multicall__, item):
-    user_del_action = None
-    if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
-        if (appclass is not None and
-            not getattr(appclass.obj, 'runappdirect', False) and
-            hasattr(appclass.obj, 'space')):
-            user_del_action = appclass.obj.space.user_del_action
-
-    if user_del_action:
-        # if leakfinder triggers leftover __del__s, ensure their
-        # enqueue_for_destruction callbacks are invoked immediately
-        # instead of scheduled for later (potentially never)
-        user_del_action._invoke_immediately = True
-    try:
-        # leakfinder
-        __multicall__.execute()
-    finally:
-        if user_del_action:
-            user_del_action._invoke_immediately = False
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
+    __multicall__.execute()
 
 
 class PyPyClassCollector(py.test.collect.Class):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -2,7 +2,7 @@
 # This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
-# Missing vs CPython: -b, -d, -x, -3
+# Missing vs CPython: -b, -d, -x
 from __future__ import print_function, unicode_literals
 USAGE1 = __doc__ = """\
 Options and arguments (and corresponding environment variables):
@@ -16,10 +16,10 @@
 -O     : skip assert statements; also PYTHONOPTIMIZE=x
 -OO    : remove docstrings when importing modules in addition to -O
 -q     : don't print version and copyright messages on interactive startup
--R     : ignored (see http://bugs.python.org/issue14621)
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE
 -S     : don't imply 'import site' on initialization
--u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x
+-u     : unbuffered binary stdout and stderr, stdin always buffered;
+         also PYTHONUNBUFFERED=x
 -v     : verbose (trace import statements); also PYTHONVERBOSE=x
          can be supplied multiple times to increase verbosity
 -V     : print the Python version number and exit (also --version)
@@ -379,6 +379,9 @@
 def end_options(options, _, iterargv):
     return list(iterargv)
 
+def ignore_option(*args):
+    pass
+
 cmdline_options = {
     # simple options just increment the counter of the options listed above
     'b': (simple_option, 'bytes_warning'),
@@ -387,7 +390,6 @@
     'E': (simple_option, 'ignore_environment'),
     'i': (simple_option, 'interactive'),
     'O': (simple_option, 'optimize'),
-    'R': (simple_option, 'hash_randomization'),
     's': (simple_option, 'no_user_site'),
     'S': (simple_option, 'no_site'),
     'u': (simple_option, 'unbuffered'),
@@ -407,6 +409,7 @@
     '--jit':     (set_jit_option,  Ellipsis),
     '-funroll-loops': (funroll_loops, None),
     '--':        (end_options,     None),
+    'R':         (ignore_option,   None),      # previously hash_randomization
     }
 
 def handle_argument(c, options, iterargv, iterarg=iter(())):
diff --git a/pypy/interpreter/astcompiler/test/test_ast.py b/pypy/interpreter/astcompiler/test/test_ast.py
--- a/pypy/interpreter/astcompiler/test/test_ast.py
+++ b/pypy/interpreter/astcompiler/test/test_ast.py
@@ -1,8 +1,8 @@
 from pypy.interpreter.astcompiler import ast
 class TestAstToObject:
     def test_types(self, space):
-        assert space.is_true(space.issubtype(
-                ast.get(space).w_Module, ast.get(space).w_mod))
+        assert space.issubtype_w(
+                ast.get(space).w_Module, ast.get(space).w_mod)
                                   
     def test_num(self, space):
         value = space.wrap(42)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -593,9 +593,6 @@
         # lives in pypy/module/exceptions, we rename it below for
         # sys.builtin_module_names
         bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
-        if sys.platform.startswith("win"):
-            self.setbuiltinmodule('_winreg')
-            bootstrap_modules.add('winreg')
         installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
@@ -1206,7 +1203,7 @@
 
     def abstract_issubclass_w(self, w_cls1, w_cls2):
         # Equivalent to 'issubclass(cls1, cls2)'.
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def abstract_isinstance_w(self, w_obj, w_cls):
         # Equivalent to 'isinstance(obj, cls)'.
@@ -1236,16 +1233,16 @@
     def exception_is_valid_obj_as_class_w(self, w_obj):
         if not self.isinstance_w(w_obj, self.w_type):
             return False
-        return self.is_true(self.issubtype(w_obj, self.w_BaseException))
+        return self.issubtype_w(w_obj, self.w_BaseException)
 
     def exception_is_valid_class_w(self, w_cls):
-        return self.is_true(self.issubtype(w_cls, self.w_BaseException))
+        return self.issubtype_w(w_cls, self.w_BaseException)
 
     def exception_getclass(self, w_obj):
         return self.type(w_obj)
 
     def exception_issubclass_w(self, w_cls1, w_cls2):
-        return self.is_true(self.issubtype(w_cls1, w_cls2))
+        return self.issubtype_w(w_cls1, w_cls2)
 
     def new_exception_class(self, *args, **kwargs):
         "NOT_RPYTHON; convenience method to create excceptions in modules"
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -539,8 +539,6 @@
         self.pending_with_disabled_del = None
 
     def perform(self, executioncontext, frame):
-        if self.finalizers_lock_count > 0:
-            return
         self._run_finalizers()
 
     @jit.dont_look_inside
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -202,7 +202,7 @@
                               newArcPair(states, EMPTY),
                               pseudoExtras, number, funny, contStr, name))
     dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
-    return DFA(dfaStates, dfaAccepts)
+    return DFA(dfaStates, dfaAccepts), dfaStates
 
 # ______________________________________________________________________
 
@@ -216,7 +216,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, "'\\")))),
                    newArcPair(states, "'"))
-    singleDFA = DFA(*nfaToDfa(states, *single))
+    states, accepts = nfaToDfa(states, *single)
+    singleDFA = DFA(states, accepts)
+    states_singleDFA = states
     states = []
     double = chain(states,
                    any(states, notGroupStr(states, '"\\')),
@@ -226,7 +228,9 @@
                              newArcPair(states, DEFAULT),
                              any(states, notGroupStr(states, '"\\')))),
                    newArcPair(states, '"'))
-    doubleDFA = DFA(*nfaToDfa(states, *double))
+    states, accepts = nfaToDfa(states, *double)
+    doubleDFA = DFA(states, accepts)
+    states_doubleDFA = states
     states = []
     single3 = chain(states,
                     any(states, notGroupStr(states, "'\\")),
@@ -241,7 +245,9 @@
                                           notChainStr(states, "''"))),
                               any(states, notGroupStr(states, "'\\")))),
                     chainStr(states, "'''"))
-    single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+    states, accepts = nfaToDfa(states, *single3)
+    single3DFA = NonGreedyDFA(states, accepts)
+    states_single3DFA = states
     states = []
     double3 = chain(states,
                     any(states, notGroupStr(states, '"\\')),
@@ -256,27 +262,34 @@
                                           notChainStr(states, '""'))),
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
-    double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    return {"'" : singleDFA,
-            '"' : doubleDFA,
-            "'''": single3DFA,
-            '"""': double3DFA}
+    states, accepts = nfaToDfa(states, *double3)
+    double3DFA = NonGreedyDFA(states, accepts)
+    states_double3DFA = states
+    return {"'" : (singleDFA, states_singleDFA),
+            '"' : (doubleDFA, states_doubleDFA),
+            "'''": (single3DFA, states_single3DFA),
+            '"""': (double3DFA, states_double3DFA)}
 
 # ______________________________________________________________________
 
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
     import textwrap
+    lines = []
     i = 0
     for line in textwrap.wrap(repr(dfa.accepts), width = 50):
         if i == 0:
-            print "accepts =", line
+            lines.append("accepts = ")
         else:
-            print "          ", line
+            lines.append("           ")
+        lines.append(line)
+        lines.append("\n")
         i += 1
     import StringIO
-    print "states = ["
-    for numstate, state in enumerate(dfa.states):
-        print "    #", numstate
+    lines.append("states = [\n")
+    for numstate, state in enumerate(states):
+        lines.append("    # ")
+        lines.append(str(numstate))
+        lines.append("\n")
         s = StringIO.StringIO()
         i = 0
         for k, v in sorted(state.items()):
@@ -299,13 +312,15 @@
         for line in text:
             line = line.replace('::', ': ')
             if i == 0:
-                print '    {' + line
+                lines.append('    {')
             else:
-                print '     ' + line
+                lines.append('     ')
+            lines.append(line)
+            lines.append('\n')
             i += 1
-    print "    ]"
-    print "%s = automata.%s(states, accepts)" % (name, dfa_class)
-    print
+    lines.append("    ]\n")
+    lines.append("%s = automata.%s(states, accepts)\n" % (name, dfa_class))
+    return ''.join(lines)
 
 def main ():
     print "# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py"
@@ -314,13 +329,17 @@
     print "#     python gendfa.py > dfa_generated.py"
     print
     print "from pypy.interpreter.pyparser import automata"
-    pseudoDFA = makePyPseudoDFA()
-    output("pseudoDFA", "DFA", pseudoDFA)
+    pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+    print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
-    output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
-    output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
-    output("singleDFA", "DFA", endDFAMap["'"])
-    output("doubleDFA", "DFA", endDFAMap['"'])
+    dfa, states = endDFAMap['"""']
+    print output("double3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'''"]
+    print output("single3DFA", "NonGreedyDFA", dfa, states)
+    dfa, states = endDFAMap["'"]
+    print output("singleDFA", "DFA", dfa, states)
+    dfa, states = endDFAMap['"']
+    print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
 
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -0,0 +1,16 @@
+from pypy.interpreter.pyparser.automata import DFA, DEFAULT
+from pypy.interpreter.pyparser.gendfa import output
+
+def test_states():
+    states = [{"\x00": 1}, {"\x01": 0}]
+    d = DFA(states[:], [False, True])
+    assert output('test', DFA, d, states) == """\
+accepts = [False, True]
+states = [
+    # 0
+    {'\\x00': 1},
+    # 1
+    {'\\x01': 0},
+    ]
+test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
+"""
diff --git a/pypy/interpreter/test/conftest.py b/pypy/interpreter/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/conftest.py
@@ -0,0 +1,13 @@
+from pypy.conftest import PYTHON3
+
+def get_banner():
+    import subprocess
+    p = subprocess.Popen([PYTHON3, "-c",
+                          "import sys; print(sys.version.splitlines()[0])"],
+                         stdout=subprocess.PIPE)
+    return p.stdout.read().rstrip()
+banner = get_banner() if PYTHON3 else "PYTHON3 not found"
+
+def pytest_report_header(config):
+    if PYTHON3:
+        return "PYTHON3: %s\n(Version %s)" % (PYTHON3, banner)
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -6,22 +6,20 @@
 import sys, os, re, runpy, subprocess
 from rpython.tool.udir import udir
 from contextlib import contextmanager
-from pypy.conftest import pypydir
+from pypy.conftest import PYTHON3, pypydir
+from pypy.interpreter.test.conftest import banner
 from lib_pypy._pypy_interact import irc_header
 
-
-python3 = os.environ.get("PYTHON3", "python3")
-
-def get_banner():
-    p = subprocess.Popen([python3, "-c",
-                          "import sys; print(sys.version.splitlines()[0])"],
-                         stdout=subprocess.PIPE)
-    return p.stdout.read().rstrip()
-banner = get_banner()
-
 app_main = os.path.join(os.path.realpath(os.path.dirname(__file__)), os.pardir, 'app_main.py')
 app_main = os.path.abspath(app_main)
 
+def get_python3():
+    if PYTHON3:
+        return PYTHON3
+    import py.test
+    py.test.fail("Test requires 'python3' (not found in PATH) or a PYTHON3 "
+                 "environment variable set")
+
 _counter = 0
 def _get_next_path(ext='.py'):
     global _counter
@@ -37,7 +35,7 @@
 def getscript_pyc(space, source):
     p = _get_next_path()
     p.write(str(py.code.Source(source)))
-    subprocess.check_call([python3, "-c", "import " + p.purebasename],
+    subprocess.check_call([get_python3(), "-c", "import " + p.purebasename],
                           env={'PYTHONPATH': str(p.dirpath())})
     # the .pyc file should have been created above
     pycache = p.dirpath('__pycache__')
@@ -99,7 +97,7 @@
                     "option %r has unexpectedly the value %r" % (key, value))
 
     def check(self, argv, env, **expected):
-        p = subprocess.Popen([python3, app_main,
+        p = subprocess.Popen([get_python3(), app_main,
                               '--argparse-only'] + list(argv),
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              env=env)
@@ -240,7 +238,7 @@
     def spawn(self, argv, env=None):
         # make sure that when we do 'import pypy' we get the correct package
         with setpythonpath():
-            return self._spawn(python3, [app_main] + argv, env=env)
+            return self._spawn(get_python3(), [app_main] + argv, env=env)
 
     def test_interactive(self):
         child = self.spawn([])
@@ -529,7 +527,7 @@
         if sys.platform == "win32":
             skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
-        pipe = subprocess.Popen([python3, app_main, "-u", "-i"],
+        pipe = subprocess.Popen([get_python3(), app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
                                 stdin=subprocess.PIPE,
                                 stderr=subprocess.STDOUT,
@@ -624,7 +622,7 @@
                 import __pypy__
             except:
                 py.test.skip('app_main cannot run on non-pypy for windows')
-        cmdline = '%s %s "%s" %s' % (python3, python_flags,
+        cmdline = '%s %s "%s" %s' % (get_python3(), python_flags,
                                      app_main, cmdline)
         print 'POPEN:', cmdline
         process = subprocess.Popen(
@@ -813,7 +811,7 @@
             time.sleep(1)
             # stdout flushed automatically here
             """)
-        cmdline = '%s -E "%s" %s' % (python3, app_main, path)
+        cmdline = '%s -E "%s" %s' % (get_python3(), app_main, path)
         print 'POPEN:', cmdline
         child_in, child_out_err = os.popen4(cmdline)
         data = child_out_err.read(11)
@@ -840,7 +838,7 @@
             if 'stderr' in streams:
                 os.close(2)
         p = subprocess.Popen(
-            [python3, app_main, "-E", "-c", code],
+            [get_python3(), app_main, "-E", "-c", code],
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -73,11 +73,10 @@
         try:
             if space.is_w(w_pretendtype, space.type(w_obj)):
                 return False     # common case: obj.__class__ is type(obj)
-            if allow_override:
-                w_result = space.issubtype_allow_override(w_pretendtype,
-                                                          w_klass_or_tuple)
-            else:
-                w_result = space.issubtype(w_pretendtype, w_klass_or_tuple)
+            if not allow_override:
+                return space.issubtype_w(w_pretendtype, w_klass_or_tuple)
+            w_result = space.issubtype_allow_override(w_pretendtype,
+                                                      w_klass_or_tuple)
         except OperationError as e:
             if e.async(space):
                 raise
@@ -130,11 +129,9 @@
 
     # -- case (type, type)
     try:
-        if allow_override:
-            w_result = space.issubtype_allow_override(w_derived,
-                                                      w_klass_or_tuple)
-        else:
-            w_result = space.issubtype(w_derived, w_klass_or_tuple)
+        if not allow_override:
+            return space.issubtype_w(w_derived, w_klass_or_tuple)
+        w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple)
     except OperationError as e:   # if one of the args was not a type, ignore it
         if not e.match(space, space.w_TypeError):
             raise       # propagate other errors
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -15,12 +15,14 @@
 
     def descr_init(self, space, w_starttype=None, w_obj_or_type=None):
         if space.is_none(w_starttype):
-            w_starttype, w_obj_or_type = _super_from_frame(space)
+            frame = space.getexecutioncontext().gettopframe()
+            w_starttype, w_obj_or_type = _super_from_frame(space, frame)
+
         if space.is_none(w_obj_or_type):
             w_type = None  # unbound super object
             w_obj_or_type = space.w_None
         else:
-            w_type = _supercheck(space, w_starttype, w_obj_or_type)
+            w_type = _super_check(space, w_starttype, w_obj_or_type)
         self.w_starttype = w_starttype
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
@@ -57,11 +59,10 @@
         # fallback to object.__getattribute__()
         return space.call_function(object_getattribute(space), self, w_name)
 
-def _super_from_frame(space):
+def _super_from_frame(space, frame):
     """super() without args -- fill in from __class__ and first local
     variable on the stack.
     """
-    frame = space.getexecutioncontext().gettopframe()
     code = frame.pycode
     if not code:
         raise oefmt(space.w_RuntimeError, "super(): no code object")
@@ -70,8 +71,9 @@
     w_obj = frame.locals_cells_stack_w[0]
     if not w_obj:
         raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
+
     for index, name in enumerate(code.co_freevars):
-        if name == "__class__":
+        if name == '__class__':
             break
     else:
         raise oefmt(space.w_RuntimeError, "super(): __class__ cell not found")
@@ -83,16 +85,16 @@
         raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
     return w_starttype, w_obj
 
-def _supercheck(space, w_starttype, w_obj_or_type):
+def _super_check(space, w_starttype, w_obj_or_type):
     """Check that the super() call makes sense. Returns a type"""
     w_objtype = space.type(w_obj_or_type)
 
-    if (space.is_true(space.issubtype(w_objtype, space.w_type)) and
-        space.is_true(space.issubtype(w_obj_or_type, w_starttype))):
+    if (space.issubtype_w(w_objtype, space.w_type) and
+        space.issubtype_w(w_obj_or_type, w_starttype)):
         # special case for class methods
         return w_obj_or_type
 
-    if space.is_true(space.issubtype(w_objtype, w_starttype)):
+    if space.issubtype_w(w_objtype, w_starttype):
         # normal case
         return w_objtype
 
@@ -103,7 +105,7 @@
             raise
         w_type = w_objtype
 
-    if space.is_true(space.issubtype(w_type, w_starttype)):
+    if space.issubtype_w(w_type, w_starttype):
         return w_type
     raise oefmt(space.w_TypeError,
                 "super(type, obj): obj must be an instance or subtype of type")
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -708,7 +708,7 @@
         w_obj_type = space.type(w_obj)
         w_type = get_w_type(space)
         return (space.is_w(w_obj_type, w_type) or
-                space.is_true(space.issubtype(w_obj_type, w_type)))
+                space.issubtype_w(w_obj_type, w_type))
     def check_exact(space, w_obj):
         "Implements the Py_Xxx_CheckExact function"
         w_obj_type = space.type(w_obj)
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -100,7 +100,7 @@
     w_type = space.gettypeobject(Module.typedef)
     w_obj_type = space.type(w_obj)
     return int(space.is_w(w_type, w_obj_type) or
-               space.is_true(space.issubtype(w_obj_type, w_type)))
+               space.issubtype_w(w_obj_type, w_type))
 
 @cpython_api([PyObject], PyObject, result_borrowed=True)
 def PyModule_GetDict(space, w_mod):
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -35,7 +35,7 @@
     w_obj_type = space.type(w_obj)
     w_type = space.gettypeobject(W_NDimArray.typedef)
     return (space.is_w(w_obj_type, w_type) or
-            space.is_true(space.issubtype(w_obj_type, w_type)))
+            space.issubtype_w(w_obj_type, w_type))
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, header=HEADER)
 def _PyArray_CheckExact(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -78,8 +78,7 @@
     args_w = space.fixedview(w_args)
     ref = make_ref(space, w_self)
     if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.is_true(space.issubtype(space.type(args_w[0]),
-                                         space.type(w_self)))):
+        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
         return space.w_NotImplemented
     Py_DecRef(space, ref)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
@@ -90,8 +89,7 @@
     args_w = space.fixedview(w_args)
     ref = make_ref(space, w_self)
     if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.is_true(space.issubtype(space.type(args_w[0]),
-                                         space.type(w_self)))):
+        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
         return space.w_NotImplemented
     Py_DecRef(space, ref)
     return generic_cpy_call(space, func_binary, args_w[0], w_self)
@@ -113,8 +111,7 @@
     args_w = space.fixedview(w_args)
     ref = make_ref(space, w_self)
     if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.is_true(space.issubtype(space.type(args_w[0]),
-                                         space.type(w_self)))):
+        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
         return space.w_NotImplemented
     Py_DecRef(space, ref)
     arg3 = space.w_None
@@ -346,8 +343,7 @@
     check_num_args(space, w_args, 1)
     w_other, = space.fixedview(w_args)
 
-    if not space.is_true(space.issubtype(space.type(w_self),
-                                         space.type(w_other))):
+    if not space.issubtype_w(space.type(w_self), space.type(w_other)):
         raise oefmt(space.w_TypeError,
                     "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'",
                     w_self, w_self, w_other)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -723,7 +723,7 @@
                 long intval;
                 PyObject *name;
 
-                if (!PyArg_ParseTuple(args, "l", &intval))
+                if (!PyArg_ParseTuple(args, "i", &intval))
                     return NULL;
 
                 IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -47,7 +47,7 @@
 def tuple_check_ref(space, ref):
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
     return (w_type is space.w_tuple or
-            space.is_true(space.issubtype(w_type, space.w_tuple)))
+            space.issubtype_w(w_type, space.w_tuple))
 
 def new_empty_tuple(space, length):
     """
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -233,7 +233,7 @@
     buffer, NULL if unicode is not a Unicode object."""
     # Don't use PyUnicode_Check, it will realize the object :-(
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
-    if not space.is_true(space.issubtype(w_type, space.w_unicode)):
+    if not space.issubtype_w(w_type, space.w_unicode):
         raise oefmt(space.w_TypeError, "expected unicode object")
     return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
 
diff --git a/pypy/module/imp/test/support.py b/pypy/module/imp/test/support.py
--- a/pypy/module/imp/test/support.py
+++ b/pypy/module/imp/test/support.py
@@ -4,8 +4,10 @@
 
     def setup_class(cls):
         space = cls.space
-        cls.w_testfn_unencodable = space.wrap(get_unencodable())
-        cls.w_special_char = space.wrap(get_special_char())
+        cls.testfn_unencodable = get_unencodable()
+        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
+        cls.special_char = get_special_char()
+        cls.w_special_char = space.wrap(cls.special_char)
 
 def get_unencodable():
     """Copy of the stdlib's support.TESTFN_UNENCODABLE:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -133,10 +133,9 @@
              line2 = "# encoding: iso-8859-1\n",
              bad = "# encoding: uft-8\n")
 
-    w_special_char = getattr(cls, 'w_special_char', None)
-    if not space.is_none(w_special_char):
-        special_char = space.unicode_w(w_special_char).encode(
-            sys.getfilesystemencoding())
+    special_char = cls.special_char
+    if special_char is not None:
+        special_char = special_char.encode(sys.getfilesystemencoding())
         p.join(special_char + '.py').write('pass')
 
     # create a .pyw file
@@ -746,54 +745,6 @@
             else:
                 raise AssertionError("should have failed")
 
-    def test_verbose_flag_1(self):
-        output = []
-        class StdErr(object):
-            def write(self, line):
-                output.append(line)
-
-        import sys, imp
-        old_flags = sys.flags
-
-        class Flags(object):
-            verbose = 1
-            def __getattr__(self, name):
-                return getattr(old_flags, name)
-
-        sys.flags = Flags()
-        sys.stderr = StdErr()
-        try:
-            import verbose1pkg.verbosemod
-        finally:
-            imp.reload(sys)
-        assert 'import verbose1pkg # ' in output[-2]
-        assert 'import verbose1pkg.verbosemod # ' in output[-1]
-
-    def test_verbose_flag_2(self):
-        output = []
-        class StdErr(object):
-            def write(self, line):
-                output.append(line)
-
-        import sys, imp
-        old_flags = sys.flags
-
-        class Flags(object):
-            verbose = 2
-            def __getattr__(self, name):
-                return getattr(old_flags, name)
-
-        sys.flags = Flags()
-        sys.stderr = StdErr()
-        try:
-            import verbose2pkg.verbosemod
-        finally:
-            imp.reload(sys)
-        assert any('import verbose2pkg # ' in line
-                   for line in output[:-2])
-        assert output[-2].startswith('# trying')
-        assert 'import verbose2pkg.verbosemod # ' in output[-1]
-
     def test_verbose_flag_0(self):
         output = []
         class StdErr(object):
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -348,8 +348,8 @@
     def descr_view(self, space, w_dtype):
         from pypy.module.micronumpy.descriptor import W_Dtype
         try:
-            subclass = space.is_true(space.issubtype(
-                w_dtype, space.gettypefor(W_NDimArray)))
+            subclass = space.issubtype_w(w_dtype,
+                                         space.gettypefor(W_NDimArray))
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 subclass = False
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -1081,7 +1081,7 @@
         if w_dtype is dtype.w_box_type:
             return _set_metadata_and_copy(space, w_metadata, dtype, copy)
         if space.isinstance_w(w_dtype, space.w_type) and \
-           space.is_true(space.issubtype(w_dtype, dtype.w_box_type)):
+           space.issubtype_w(w_dtype, dtype.w_box_type):
             return _set_metadata_and_copy( space, w_metadata,
                             W_Dtype(dtype.itemtype, w_dtype, elsize=0), copy)
     if space.isinstance_w(w_dtype, space.w_type):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -969,8 +969,7 @@
     def descr_view(self, space, w_dtype=None, w_type=None):
         if not w_type and w_dtype:
             try:
-                if space.is_true(space.issubtype(
-                        w_dtype, space.gettypefor(W_NDimArray))):
+                if space.issubtype_w(w_dtype, space.gettypefor(W_NDimArray)):
                     w_type = w_dtype
                     w_dtype = None
             except OperationError as e:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -66,10 +66,10 @@
     lhs_for_subtype = w_lhs
     rhs_for_subtype = w_rhs
     #it may be something like a FlatIter, which is not an ndarray
-    if not space.is_true(space.issubtype(lhs_type, w_ndarray)):
+    if not space.issubtype_w(lhs_type, w_ndarray):
         lhs_type = space.type(w_lhs.base)
         lhs_for_subtype = w_lhs.base
-    if not space.is_true(space.issubtype(rhs_type, w_ndarray)):
+    if not space.issubtype_w(rhs_type, w_ndarray):
         rhs_type = space.type(w_rhs.base)
         rhs_for_subtype = w_rhs.base
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -166,7 +166,8 @@
 def path_or_fd(allow_fd=True):
     return _PathOrFd if allow_fd else _JustPath
 
-DEFAULT_DIR_FD = getattr(rposix, 'AT_FDCWD', -100)
+_HAVE_AT_FDCWD = getattr(rposix, 'AT_FDCWD', None) is not None
+DEFAULT_DIR_FD = rposix.AT_FDCWD if _HAVE_AT_FDCWD else -100
 DIR_FD_AVAILABLE = False
 
 @specialize.arg(2)
@@ -196,7 +197,7 @@
 
 class _DirFD_Unavailable(Unwrapper):
     def unwrap(self, space, w_value):
-        dir_fd = unwrap_fd(space, w_value)
+        dir_fd = _unwrap_dirfd(space, w_value)
         if dir_fd == DEFAULT_DIR_FD:
             return dir_fd
         raise oefmt(space.w_NotImplementedError,
@@ -222,11 +223,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
-        else:
+        if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             fd = rposix.openat(path, flags, mode, dir_fd)
+        else:
+            fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     return space.wrap(fd)
@@ -555,7 +556,7 @@
     dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=kwonly(bool),
     follow_symlinks=kwonly(bool))
 def access(space, w_path, mode,
-        dir_fd=DEFAULT_DIR_FD, effective_ids=True, follow_symlinks=True):
+        dir_fd=DEFAULT_DIR_FD, effective_ids=False, follow_symlinks=True):
     """\
 access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
 
@@ -585,12 +586,14 @@
             raise argument_unavailable(space, "access", "effective_ids")
 
     try:
-        if dir_fd == DEFAULT_DIR_FD and follow_symlinks and not effective_ids:
-            ok = dispatch_filename(rposix.access)(space, w_path, mode)
-        else:
+        if (rposix.HAVE_FACCESSAT and
+            (dir_fd != DEFAULT_DIR_FD or not follow_symlinks or
+             effective_ids)):
             path = space.fsencode_w(w_path)
             ok = rposix.faccessat(path, mode,
                 dir_fd, effective_ids, follow_symlinks)
+        else:
+            ok = dispatch_filename(rposix.access)(space, w_path, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -635,11 +638,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.unlink)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=False)
+        else:
+            dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -654,11 +657,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.unlink)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=False)
+        else:
+            dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -721,11 +724,11 @@
 
 The mode argument is ignored on Windows."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.mkdir)(space, w_path, mode)
-        else:
+        if rposix.HAVE_MKDIRAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.mkdirat(path, mode, dir_fd)
+        else:
+            dispatch_filename(rposix.mkdir)(space, w_path, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -740,11 +743,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.rmdir)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=True)
+        else:
+            dispatch_filename(rposix.rmdir)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -976,7 +979,8 @@
 src_dir_fd and dst_dir_fd, may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD):
+        if (rposix.HAVE_RENAMEAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD)):
             src = space.fsencode_w(w_src)
             dst = space.fsencode_w(w_dst)
             rposix.renameat(src, dst, src_dir_fd, dst_dir_fd)
@@ -999,7 +1003,8 @@
 src_dir_fd and dst_dir_fd, may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD):
+        if (rposix.HAVE_RENAMEAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD)):
             src = space.fsencode_w(w_src)
             dst = space.fsencode_w(w_dst)
             rposix.renameat(src, dst, src_dir_fd, dst_dir_fd)
@@ -1110,8 +1115,9 @@
   platform.  If they are unavailable, using them will raise a
   NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD
-                or not follow_symlinks):
+        if (rposix.HAVE_LINKAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD
+             or not follow_symlinks)):
             rposix.linkat(src, dst, src_dir_fd, dst_dir_fd, follow_symlinks)
         else:
             rposix.link(src, dst)
@@ -1136,12 +1142,12 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename_2(rposix.symlink)(space, w_src, w_dst)
-        else:
+        if rposix.HAVE_SYMLINKAT and dir_fd != DEFAULT_DIR_FD:
             src = space.fsencode_w(w_src)
             dst = space.fsencode_w(w_dst)
             rposix.symlinkat(src, dst, dir_fd)
+        else:
+            dispatch_filename_2(rposix.symlink)(space, w_src, w_dst)
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -1159,10 +1165,10 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
+        if rposix.HAVE_READLINKAT and dir_fd != DEFAULT_DIR_FD:
+            result = call_rposix(rposix.readlinkat, path, dir_fd)
+        else:
             result = call_rposix(rposix.readlink, path)
-        else:
-            result = call_rposix(rposix.readlinkat, path, dir_fd)
     except OSError as e:
         raise wrap_oserror2(space, e, path.w_path)
     w_result = space.wrapbytes(result)
@@ -1442,31 +1448,32 @@
             # see comment above
             raise wrap_oserror(space, e)
 
+    if (rposix.HAVE_LUTIMES and
+        (dir_fd == DEFAULT_DIR_FD and not follow_symlinks)):
+        path_b = path.as_bytes
+        if path_b is None:
+            raise oefmt(space.w_NotImplementedError,
+                        "utime: unsupported value for 'path'")
+        try:
+            if now:
+                rposix.lutimes(path_b, None)
+            else:
+                rposix.lutimes(path_b, (atime_s, atime_ns))
+            return
+        except OSError as e:
+            # see comment above
+            raise wrap_oserror(space, e)
+
+    # XXX: missing utime_dir_fd support
+
     if not follow_symlinks:
         raise argument_unavailable(space, "utime", "follow_symlinks")
 
-    if not space.is_w(w_ns, space.w_None):
-        raise oefmt(space.w_NotImplementedError,
-            "utime: 'ns' unsupported on this platform on PyPy")
-    if now:
-        try:
+    try:
+        if now:
             call_rposix(utime_now, path, None)
-        except OSError as e:
-            # see comment above
-            raise wrap_oserror(space, e)
-    try:
-        msg = "utime() arg 2 must be a tuple (atime, mtime) or None"
-        args_w = space.fixedview(w_times)
-        if len(args_w) != 2:
-            raise oefmt(space.w_TypeError, msg)
-        actime = space.float_w(args_w[0], allow_conversion=False)
-        modtime = space.float_w(args_w[1], allow_conversion=False)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        raise oefmt(space.w_TypeError, msg)
-    try:
-        call_rposix(rposix.utime, path, (actime, modtime))
+        else:
+            call_rposix(rposix.utime, path, (atime_s, mtime_s))
     except OSError as e:
         # see comment above
         raise wrap_oserror(space, e)
diff --git a/pypy/module/posix/test/test_interp_posix.py b/pypy/module/posix/test/test_interp_posix.py
--- a/pypy/module/posix/test/test_interp_posix.py
+++ b/pypy/module/posix/test/test_interp_posix.py
@@ -1,8 +1,6 @@
 import sys
 
 import py
-from hypothesis import given
-from hypothesis.strategies import integers
 
 from rpython.tool.udir import udir
 from pypy.conftest import pypydir
@@ -44,12 +42,20 @@
     w_time = space.wrap(123.456)
     assert convert_seconds(space, w_time) == (123, 456000000)
 
- at given(s=integers(min_value=-2**30, max_value=2**30),
-       ns=integers(min_value=0, max_value=10**9))
-def test_convert_seconds_full(space, s, ns):
-    w_time = space.wrap(s + ns * 1e-9)
-    sec, nsec = convert_seconds(space, w_time)
-    assert 0 <= nsec < 1e9
-    MAX_ERR = 1e9 / 2**23 + 1  # nsec has 53 - 30 = 23 bits of precisin
-    err = (sec * 10**9 + nsec) - (s * 10**9 + ns)
-    assert -MAX_ERR < err < MAX_ERR
+def test_convert_seconds_full(space):
+    try:
+        from hypothesis import given
+        from hypothesis.strategies import integers
+    except ImportError:
+        py.test.skip("hypothesis not found")
+
+    @given(s=integers(min_value=-2**30, max_value=2**30),
+           ns=integers(min_value=0, max_value=10**9))
+    def _test_convert_seconds_full(space, s, ns):
+        w_time = space.wrap(s + ns * 1e-9)
+        sec, nsec = convert_seconds(space, w_time)
+        assert 0 <= nsec < 1e9
+        MAX_ERR = 1e9 / 2**23 + 1  # nsec has 53 - 30 = 23 bits of precisin
+        err = (sec * 10**9 + nsec) - (s * 10**9 + ns)
+        assert -MAX_ERR < err < MAX_ERR
+    _test_convert_seconds_full(space)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -738,14 +738,16 @@
     LPDWORD = rwin32.LPDWORD
     _GetSystemTimeAdjustment = rwin32.winexternal(
                                             'GetSystemTimeAdjustment',
-                                            [LPDWORD, LPDWORD, rffi.LPBOOL], 
+                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
                                             rffi.INT)
 
     def monotonic(space, w_info=None):
         result = 0
         if HAS_GETTICKCOUNT64:
+            print('has count64'.encode('ascii')) 
             result = _GetTickCount64() * 1e-3
         else:
+            print("nocount64")
             ticks = _GetTickCount()
             if ticks < time_state.last_ticks:
                 time_state.n_overflow += 1
@@ -762,9 +764,11 @@
                 space.setattr(w_info, space.wrap("implementation"),
                               space.wrap("GetTickCount()"))
             resolution = 1e-7
-            with lltype.scoped_alloc(rwin32.LPDWORD) as time_adjustment, \
-                 lltype.scoped_alloc(rwin32.LPDWORD) as time_increment, \
-                 lltype.scoped_alloc(rwin32.LPBOOL) as is_time_adjustment_disabled:
+            print("creating a thing".encode("ascii"))
+            with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_adjustment, \
+                 lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_increment, \
+                 lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
+                print("CREATED".encode("ascii"))
                 ok = _GetSystemTimeAdjustment(time_adjustment,
                                               time_increment,
                                               is_time_adjustment_disabled)
@@ -772,8 +776,8 @@
                     # Is this right? Cargo culting...
                     raise wrap_windowserror(space,
                         rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
-                resolution = resolution * time_increment
-                
+                resolution = resolution * time_increment[0]
+            print("out of with".encode("ascii"))
             space.setattr(w_info, space.wrap("monotonic"), space.w_True)
             space.setattr(w_info, space.wrap("adjustable"), space.w_False)
             space.setattr(w_info, space.wrap("resolution"),
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -347,7 +347,7 @@
             w_right_src, w_right_impl = space.lookup_in_type_where(w_typ2, '__rpow__')
             # sse binop_impl
             if (w_left_src is not w_right_src
-                and space.is_true(space.issubtype(w_typ2, w_typ1))):
+                and space.issubtype_w(w_typ2, w_typ1)):
                 if (w_left_src and w_right_src and
                     not space.abstract_issubclass_w(w_left_src, w_right_src) and
                     not space.abstract_issubclass_w(w_typ1, w_right_src)):
@@ -454,8 +454,11 @@
         assert isinstance(w_result, W_AbstractIntObject)
         return w_result.descr_hash(space)
 
+    def issubtype_w(space, w_sub, w_type):
+        return space._type_issubtype(w_sub, w_type)
+
     def issubtype(space, w_sub, w_type):
-        return space._type_issubtype(w_sub, w_type)
+        return space.wrap(space._type_issubtype(w_sub, w_type))
 
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
@@ -524,7 +527,7 @@
                 if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat
                                     and not w_typ2.flag_sequence_bug_compat)
                         # the non-bug-compat part is the following check:
-                        or space.is_true(space.issubtype(w_typ2, w_typ1))):
+                        or space.issubtype_w(w_typ2, w_typ1)):
                     if (not space.abstract_issubclass_w(w_left_src, w_right_src) and
                         not space.abstract_issubclass_w(w_typ1, w_right_src)):
                         w_obj1, w_obj2 = w_obj2, w_obj1
@@ -579,7 +582,7 @@
             # if the type is the same, then don't reverse: try
             # left first, right next.
             pass
-        elif space.is_true(space.issubtype(w_typ2, w_typ1)):
+        elif space.issubtype_w(w_typ2, w_typ1):
             # if typ2 is a subclass of typ1.
             w_obj1, w_obj2 = w_obj2, w_obj1
             w_left_impl, w_right_impl = w_right_impl, w_left_impl
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -291,6 +291,11 @@
     def type(self, w_obj):
         return w_some_type()
 
+    def issubtype_w(self, w_sub, w_type):
+        is_root(w_sub)
+        is_root(w_type)
+        return NonConstant(True)
+
     def isinstance_w(self, w_inst, w_type):
         is_root(w_inst)
         is_root(w_type)
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -509,7 +509,7 @@
         if not isinstance(w_obj, W_ComplexObject):
             raise oefmt(space.w_TypeError, "descriptor is for 'complex'")
         return space.newfloat(getattr(w_obj, name))
-    return GetSetProperty(fget, doc=doc)
+    return GetSetProperty(fget, doc=doc, cls=W_ComplexObject)
 
 W_ComplexObject.typedef = TypeDef("complex",
     __doc__ = """complex(real[, imag]) -> complex number
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -50,7 +50,7 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_true(space.issubtype(w_lookup_type, space.w_unicode)):
+        if space.issubtype_w(w_lookup_type, space.w_unicode):
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -650,7 +650,7 @@
 
     def _type_issubtype(self, w_sub, w_type):
         if isinstance(w_sub, W_TypeObject) and isinstance(w_type, W_TypeObject):
-            return self.wrap(w_sub.issubtype(w_type))
+            return w_sub.issubtype(w_type)
         raise oefmt(self.w_TypeError, "need type objects")
 
     @specialize.arg_or_var(2)
diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py
--- a/pypy/objspace/std/transparent.py
+++ b/pypy/objspace/std/transparent.py
@@ -52,15 +52,15 @@
         raise oefmt(space.w_TypeError, "controller should be function")
 
     if isinstance(w_type, W_TypeObject):
-        if space.is_true(space.issubtype(w_type, space.gettypeobject(Function.typedef))):
+        if space.issubtype_w(w_type, space.gettypeobject(Function.typedef)):
             return W_TransparentFunction(space, w_type, w_controller)
-        if space.is_true(space.issubtype(w_type, space.gettypeobject(PyTraceback.typedef))):
+        if space.issubtype_w(w_type, space.gettypeobject(PyTraceback.typedef)):
             return W_TransparentTraceback(space, w_type, w_controller)
-        if space.is_true(space.issubtype(w_type, space.gettypeobject(PyFrame.typedef))):
+        if space.issubtype_w(w_type, space.gettypeobject(PyFrame.typedef)):
             return W_TransparentFrame(space, w_type, w_controller)
-        if space.is_true(space.issubtype(w_type, space.gettypeobject(GeneratorIterator.typedef))):
+        if space.issubtype_w(w_type, space.gettypeobject(GeneratorIterator.typedef)):
             return W_TransparentGenerator(space, w_type, w_controller)
-        if space.is_true(space.issubtype(w_type, space.gettypeobject(PyCode.typedef))):
+        if space.issubtype_w(w_type, space.gettypeobject(PyCode.typedef)):
             return W_TransparentCode(space, w_type, w_controller)
         if w_type.layout.typedef is space.w_object.layout.typedef:
             return W_Transparent(space, w_type, w_controller)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -445,7 +445,7 @@
         cached_version_tag = cache.versions[method_hash]
         if cached_version_tag is version_tag:
             cached_name = cache.names[method_hash]
-            if cached_name is name:
+            if cached_name == name:
                 tup = cache.lookup_where[method_hash]
                 if space.config.objspace.std.withmethodcachecounter:
                     cache.hits[name] = cache.hits.get(name, 0) + 1
@@ -710,9 +710,9 @@
     w_winner = w_metaclass
     for base in bases_w:
         w_typ = space.type(base)
-        if space.is_true(space.issubtype(w_winner, w_typ)):
+        if space.issubtype_w(w_winner, w_typ):
             continue
-        if space.is_true(space.issubtype(w_typ, w_winner)):
+        if space.issubtype_w(w_typ, w_winner):
             w_winner = w_typ
             continue
         msg = ("metaclass conflict: the metaclass of a derived class must be "
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -25,7 +25,7 @@
             pass
         return Base, Sub""")
         w_base, w_sub = space.unpackiterable(w_tup)
-        assert space.is_true(space.issubtype(w_sub, w_base))
+        assert space.issubtype_w(w_sub, w_base)
         w_inst = space.call_function(w_sub)
         assert space.isinstance_w(w_inst, w_base)
 
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -1,4 +1,4 @@
-import os, random, struct
+import sys, os, random, struct
 import py
 from rpython.jit.backend.x86 import rx86
 from rpython.rlib.rarithmetic import intmask
@@ -257,6 +257,9 @@
         g.close()
         error = [line for line in got.splitlines() if 'error' in line.lower()]
         if error:
+            if (sys.maxint <= 2**32 and
+                    'no compiled in support for x86_64' in error[0]):
+                py.test.skip(error)
             raise Exception("Assembler got an error: %r" % error[0])
         error = [line for line in got.splitlines()
                  if 'warning' in line.lower()]
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1219,21 +1219,14 @@
         if times is None:
             error = c_utime(path, lltype.nullptr(UTIMBUFP.TO))
         else:
-            actime, modtime = times
             if HAVE_UTIMES:
-                import math
-                l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw')
-                fracpart, intpart = math.modf(actime)
-                rffi.setintfield(l_times[0], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[0], 'c_tv_usec', int(fracpart * 1e6))
-                fracpart, intpart = math.modf(modtime)
-                rffi.setintfield(l_times[1], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[1], 'c_tv_usec', int(fracpart * 1e6))
-                error = c_utimes(path, l_times)
-                lltype.free(l_times, flavor='raw')
+                with lltype.scoped_alloc(TIMEVAL2P.TO, 2) as l_timeval2p:
+                    times_to_timeval2p(times, l_timeval2p)
+                    error = c_utimes(path, l_timeval2p)
             else:
                 # we only have utime(), which does not allow
                 # sub-second resolution
+                actime, modtime = times
                 l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw')
                 l_utimbuf.c_actime  = rffi.r_time_t(actime)
                 l_utimbuf.c_modtime = rffi.r_time_t(modtime)
@@ -1276,6 +1269,17 @@
             lltype.free(atime, flavor='raw')
             lltype.free(mtime, flavor='raw')
 
+def times_to_timeval2p(times, l_timeval2p):
+    actime, modtime = times
+    _time_to_timeval(actime, l_timeval2p[0])
+    _time_to_timeval(modtime, l_timeval2p[1])
+
+def _time_to_timeval(t, l_timeval):
+    import math
+    fracpart, intpart = math.modf(t)
+    rffi.setintfield(l_timeval, 'c_tv_sec', int(intpart))
+    rffi.setintfield(l_timeval, 'c_tv_usec', int(fracpart * 1e6))
+
 if not _WIN32:
     TMSP = lltype.Ptr(TMS)
     c_times = external('times', [TMSP], CLOCK_T,
@@ -1763,6 +1767,7 @@
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
         includes=['sys/stat.h',
+                  'sys/time.h',
                   'unistd.h',
                   'fcntl.h'],
     )
@@ -1918,6 +1923,21 @@
         lltype.free(l_times, flavor='raw')
         handle_posix_error('utimensat', error)
 
+if HAVE_LUTIMES:
+    c_lutimes = external('lutimes',
+        [rffi.CCHARP, TIMEVAL2P], rffi.INT,
+        save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @specialize.argtype(1)
+    def lutimes(pathname, times):
+        if times is None:
+            error = c_lutimes(pathname, lltype.nullptr(TIMEVAL2P.TO))
+        else:
+            with lltype.scoped_alloc(TIMEVAL2P.TO, 2) as l_timeval2p:
+                times_to_timeval2p(times, l_timeval2p)
+                error = c_lutimes(pathname, l_timeval2p)
+        handle_posix_error('lutimes', error)
+
 if HAVE_MKDIRAT:
     c_mkdirat = external('mkdirat',
         [rffi.INT, rffi.CCHARP, rffi.INT], rffi.INT,

From pypy.commits at gmail.com  Mon Jun 20 18:16:50 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Testing
Message-ID: <57686b52.4aa71c0a.c06a3.355f@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85276:4e492dd30bf2
Date: 2016-05-24 22:06 -0400
http://bitbucket.org/pypy/pypy/changeset/4e492dd30bf2/

Log:	Testing

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -39,7 +39,9 @@
         includes = ['windows.h'],
         post_include_bits = [
             "RPY_EXTERN\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
+            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"
+            "ULONGLONG pypy_GetTickCount64(FARPROC address);"
+            "],
         separate_module_sources=['''
             static HANDLE interrupt_event;
 
@@ -60,6 +62,12 @@
                 return SetConsoleCtrlHandler(CtrlHandlerRoutine, TRUE);
             }
 
+            ULONGLONG pypy_GetTickCount64(FARPROC address) {
+                ULONGLONG (WINAPI *func)();
+                *(FARPROC*)&func = address;
+                return func();
+            }   
+
         '''],
         )
     _setCtrlHandlerRoutine = rffi.llexternal(
@@ -68,6 +76,21 @@
         compilation_info=eci,
         save_err=rffi.RFFI_SAVE_LASTERROR)
 
+    pypy_GetTickCount64 = rffi.llexternal(
+        'pypy_GetTickCount64',
+        [rffi.VOIDP],
+        rffi.ULONGLONG, compilation_info=eci)
+
+    try:
+        hKernel32 = GetModuleHandle("KERNEL32")
+        try:
+            _GetTickCount64_handle = dlsym(hKernel32, 'GetTickCount64')
+            def _GetTickCount64():
+                return pypy_GetTickCount64(_GetTickCount64_handle)
+        except KeyError:
+            _GetTickCount64_handle = lltype.nullptr(rffi.VOIDP.TO))
+
+    HAS_GETTICKCOUNT64 = pypy_GetTickCount64 != lltype.nullptr(rffi.VOIDP.TO))
     class GlobalState:
         def __init__(self):
             self.init()
@@ -732,14 +755,6 @@
                                             'GetSystemTimeAdjustment',
                                             [LPDWORD, LPDWORD, rwin32.LPBOOL], 
                                             rffi.INT)
-    from rpython.rlib.rdynload import GetModuleHandle, dlsym
-    hKernel32 = GetModuleHandle("KERNEL32")
-    try:
-        _GetTickCount64 = dlsym(hKernel32, 'GetTickCount64')
-        HAS_GETTICKCOUNT64 = True
-    except KeyError:
-        HAS_GETTICKCOUNT64 = False
-
     def monotonic(space, w_info=None):
         result = 0
         if HAS_GETTICKCOUNT64:

From pypy.commits at gmail.com  Mon Jun 20 18:16:51 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Commit what I have for now.
Message-ID: <57686b53.43921c0a.7282.31b7@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85277:2932c4d0829b
Date: 2016-05-27 01:22 -0400
http://bitbucket.org/pypy/pypy/changeset/2932c4d0829b/

Log:	Commit what I have for now.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -39,9 +39,9 @@
         includes = ['windows.h'],
         post_include_bits = [
             "RPY_EXTERN\n"
-            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"
-            "ULONGLONG pypy_GetTickCount64(FARPROC address);"
-            "],
+            "BOOL pypy_timemodule_setCtrlHandler(HANDLE event);\n"
+            "RPY_EXTERN ULONGLONG pypy_GetTickCount64(FARPROC address);"
+        ],
         separate_module_sources=['''
             static HANDLE interrupt_event;
 
@@ -81,16 +81,16 @@
         [rffi.VOIDP],
         rffi.ULONGLONG, compilation_info=eci)
 
+    from rpython.rlib.rdynload import GetModuleHandle, dlsym
+    hKernel32 = GetModuleHandle("KERNEL32")
     try:
-        hKernel32 = GetModuleHandle("KERNEL32")
-        try:
-            _GetTickCount64_handle = dlsym(hKernel32, 'GetTickCount64')
-            def _GetTickCount64():
-                return pypy_GetTickCount64(_GetTickCount64_handle)
-        except KeyError:
-            _GetTickCount64_handle = lltype.nullptr(rffi.VOIDP.TO))
+        _GetTickCount64_handle = dlsym(hKernel32, 'GetTickCount64')
+        def _GetTickCount64():
+            return pypy_GetTickCount64(_GetTickCount64_handle)
+    except KeyError:
+        _GetTickCount64_handle = lltype.nullptr(rffi.VOIDP.TO)
 
-    HAS_GETTICKCOUNT64 = pypy_GetTickCount64 != lltype.nullptr(rffi.VOIDP.TO))
+    HAS_GETTICKCOUNT64 = _GetTickCount64_handle != lltype.nullptr(rffi.VOIDP.TO)
     class GlobalState:
         def __init__(self):
             self.init()
@@ -149,7 +149,6 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
-    has_gettickcount64 = platform.Has("GetTickCount64")
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
@@ -182,6 +181,13 @@
         ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
+    # TODO: Figure out how to implement this...
+    CConfig.ULARGE_INTEGER = platform.Struct("struct ULARGE_INTEGER", [
+        ("tm_sec", rffi.INT),
+        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
+        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
+        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
+
 if _MACOSX:
     CConfig.TIMEBASE_INFO = platform.Struct("struct mach_timebase_info", [
         ("numer", rffi.UINT),
@@ -222,7 +228,41 @@
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
 if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
+    
+    c_gettimeofday = external('gettimeofday', 
+                              [CConfig.timeval,
+rffi.VOIDP], 
+                              rffi.INT)
+    if _WIN:
+       GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime', 
+                                          [rwin32.FILETIME], 
+                                          lltype.VOID)
+        def gettimeofday(space, w_info=None):
+            with lltype.scoped_alloc(rwin32.FILETIME) as system_time, 
+                GetSystemTimeAsFileTime(system_time)
+                
+
+                seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
+
+            return space.wrap(seconds)
+    else:
+        def gettimeofday(space, w_info=None):
+            with lltype.scoped_alloc(CConfig.timeval) as timeval:
+                ret = c_gettimeofday(timeval, rffi.NULL)
+                if ret != 0:
+                    raise exception_from_saved_errno(space, space.w_OSError)
+                
+                space.setattr(w_info, space.wrap("implementation"),
+                              space.wrap("gettimeofday()"))
+                space.setattr(w_info, space.wrap("resolution"), 1e-6)
+                space.setattr(w_info, space.wrap("monotonic"), space.w_False)
+                space.setattr(w_info, space.wrap("adjustable"), space.w_True)
+
+                seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
+            return space.wrap(seconds)
+        
+
+u
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
 c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P,
@@ -538,16 +578,30 @@
     secs = pytime.time()
     return space.wrap(secs)
 
-# TODO: Remember what this is for...
 def get_time_time_clock_info(space, w_info):
     # Can't piggy back on time.time because time.time delegates to the 
     # host python's time.time (so we can't see the internals)
     if HAS_CLOCK_GETTIME:
-        try:
-            res = clock_getres(space, cConfig.CLOCK_REALTIME)
-        except OperationError:
-            res = 1e-9
-    #else: ???
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            ret = c_clock_gettime(cConfig.CLOCK_REALTIME, timespec)
+            if ret != 0:
+                raise exception_from_saved_errno(space, space.w_OSError)
+            space.setattr(w_info, space.wrap("monotonic"), space.w_False)
+            space.setattr(w_info, space.wrap("implementation"),
+                          space.wrap("clock_gettime(CLOCK_REALTIME)"))
+            space.setattr(w_info, space.wrap("adjustable"), space.w_True)
+            try:
+                res = clock_getres(space, cConfig.CLOCK_REALTIME)
+            except OperationError:
+                res = 1e-9
+           
+            space.setattr(w_info, space.wrap("resolution"),
+                          res)
+            secs = _timespec_to_seconds(timespec)
+            return secs
+    else:
+        return gettimeofday(w_info)
+        
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -840,7 +894,7 @@
                 try:
                     space.setattr(w_info, space.wrap("resolution"),
                                   space.wrap(clock_getres(space, cConfig.CLOCK_HIGHRES)))
-                except OSError:
+                except OperationError:
                     space.setattr(w_info, space.wrap("resolution"),
                                   space.wrap(1e-9))
                 
@@ -855,7 +909,7 @@
                 try:
                     space.setattr(w_info, space.wrap("resolution"),
                                   space.wrap(clock_getres(space, cConfig.CLOCK_MONOTONIC)))
-                except OSError:
+                except OperationError:
                     space.setattr(w_info, space.wrap("resolution"),
                                   space.wrap(1e-9))
 

From pypy.commits at gmail.com  Mon Jun 20 18:16:57 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merging with upstream.
Message-ID: <57686b59.49c51c0a.69fe2.ffffef35@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85280:853abe86c4e9
Date: 2016-05-30 12:46 -0400
http://bitbucket.org/pypy/pypy/changeset/853abe86c4e9/

Log:	Merging with upstream.

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -8,3 +8,28 @@
 .. branch: py3k-memoryview
 
 Implement new memoryview features.
+
+.. branch: py3.3
+
+.. branch: py3.3-hashfix
+
+Use intobject hash function for specialisedtuple
+
+.. branch: follow_symlinks
+
+Add support for dir_fd and follow_symlinks in posix.stat()
+
+.. branch: stat_ns
+
+Implement the st_xtime_ns fields in stat_result()
+
+.. branch: 33_fix_itertools
+
+Add pickling support for the itertools classes
+
+.. branch: py3k-update
+
+.. branch: py3k-get_clock_info
+
+.. branch: py3k-update
+
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -67,10 +67,11 @@
         uni = runicode.str_decode_utf_8(
             bytes, len(bytes), 'surrogateescape',
             errorhandler=state.decode_error_handler)[0]
-    elif state.codec_need_encodings:
-        # bootstrap check: if the filesystem codec is implemented in
-        # Python we cannot use it before the codecs are ready. use the
-        # locale codec instead
+    elif space.sys.filesystemencoding is None or state.codec_need_encodings:
+        # bootstrap check: if the filesystemencoding isn't initialized
+        # or the filesystem codec is implemented in Python we cannot
+        # use it before the codecs are ready. use the locale codec
+        # instead
         from pypy.module._codecs.locale import (
             str_decode_locale_surrogateescape)
         bytes = space.bytes_w(w_string)
@@ -95,10 +96,11 @@
         bytes = runicode.unicode_encode_utf_8(
             uni, len(uni), 'surrogateescape',
             errorhandler=state.encode_error_handler)
-    elif state.codec_need_encodings:
-        # bootstrap check: if the filesystem codec is implemented in
-        # Python we cannot use it before the codecs are ready. use the
-        # locale codec instead
+    elif space.sys.filesystemencoding is None or state.codec_need_encodings:
+        # bootstrap check: if the filesystemencoding isn't initialized
+        # or the filesystem codec is implemented in Python we cannot
+        # use it before the codecs are ready. use the locale codec
+        # instead
         from pypy.module._codecs.locale import (
             unicode_encode_locale_surrogateescape)
         uni = space.unicode_w(w_uni)
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -587,9 +587,9 @@
         raises(TypeError, min, 1, 2, key=lambda x: x, bar=2)
         assert type(min(1, 1.0)) is int
         assert type(min(1.0, 1)) is float
-        assert type(min(1, 1.0, 1L)) is int
-        assert type(min(1.0, 1L, 1)) is float
-        assert type(min(1L, 1, 1.0)) is long
+        assert type(min(1, 1.0, 1)) is int
+        assert type(min(1.0, 1, 1)) is float
+        assert type(min(1, 1, 1.0)) is int
 
     def test_max(self):
         assert max(1, 2) == 2
@@ -599,6 +599,6 @@
         raises(TypeError, max, 1, 2, key=lambda x: x, bar=2)
         assert type(max(1, 1.0)) is int
         assert type(max(1.0, 1)) is float
-        assert type(max(1, 1.0, 1L)) is int
-        assert type(max(1.0, 1L, 1)) is float
-        assert type(max(1L, 1, 1.0)) is long
+        assert type(max(1, 1.0, 1)) is int
+        assert type(max(1.0, 1, 1)) is float
+        assert type(max(1, 1, 1.0)) is int

From pypy.commits at gmail.com  Mon Jun 20 18:17:01 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:17:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add fallbacks to gettimeofday.
Message-ID: <57686b5d.0ed11c0a.a826e.ffffa561@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85282:d77c43d58ade
Date: 2016-06-03 00:43 -0400
http://bitbucket.org/pypy/pypy/changeset/d77c43d58ade/

Log:	Add fallbacks to gettimeofday.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.rtime import win_perf_counter
+from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat
+from rpython.rlib.rtime import (win_perf_counter, TIMEB, c_ftime,
+                                GETTIMEOFDAY_NO_TZ, TIMEVAL)
 from rpython.rlib import rposix, rtime
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 import math
@@ -234,48 +235,68 @@
 
 if _WIN:
     GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime',
-                                      [rwin32.FILETIME],
-                                       rffi.VOIDP)
+                                      [lltype.Ptr(rwin32.FILETIME)],
+                                       lltype.Void)
     LPDWORD = rwin32.LPDWORD
     _GetSystemTimeAdjustment = rwin32.winexternal(
                                             'GetSystemTimeAdjustment',
                                             [LPDWORD, LPDWORD, rwin32.LPBOOL], 
                                             rffi.INT)
     def gettimeofday(space, w_info=None):
-        with lltype.scoped_alloc(rwin32.FILETIME) as system_time:#, \
-             #lltype.scoped_alloc(rffi.ULONGLONG) as microseconds:
-
+        with lltype.scoped_alloc(rwin32.FILETIME) as system_time:
             GetSystemTimeAsFileTime(system_time)
             quad_part = (system_time.c_dwLowDateTime |
-                         system_time.c_dwHighDateTime << 32)
-            microseconds = quad_part / 10 - 11644473600000000
+                         (r_ulonglong(system_time.c_dwHighDateTime) << 32))
+            # 11,644,473,600,000,000: number of microseconds between
+            # the 1st january 1601 and the 1st january 1970 (369 years + 80 leap
+            # days).
+            offset = (r_ulonglong(16384) * r_ulonglong(27) * r_ulonglong(390625)
+                     * r_ulonglong(79) * r_ulonglong(853))
+            microseconds = quad_part / 10 - offset
             tv_sec = microseconds / 1000000
             tv_usec = microseconds % 1000000
             if w_info:
-                with lltype.scoped_alloc(rwin32.DWORD) as time_adjustment, \
-                     lltype.scoped_alloc(rwin32.DWORD) as time_increment, \
-                     lltype.scoped_alloc(rwin32.BOOL) as is_time_adjustment_disabled:
-                    w_info.implementation = "GetSystemTimeAsFileTime()"
-                    w_info.monotonic = space.w_False
+                with lltype.scoped_alloc(LPDWORD.TO, 1) as time_adjustment, \
+                     lltype.scoped_alloc(LPDWORD.TO, 1) as time_increment, \
+                     lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
                     _GetSystemTimeAdjustment(time_adjustment, time_increment,
                                              is_time_adjustment_disabled)
-                    w_info.resolution = time_increment * 1e-7
-                    w_info.adjustable = space.w_True
-                    # TODO: Check that the 'casting' is correct here
-                    return space.wrap(tv_sec + tv_usec * 1e-6)
-
+                    
+                    _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
+                             time_increment[0] * 1e-7, False, True)
+            # The explicit float call can be eliminated once we merge in
+            # 9f2746f3765c from default
+            return space.wrap(float(tv_sec) + tv_usec * 1e-6)
 else:
-    c_gettimeofday = external('gettimeofday',
-                              [cConfig.timeval, rffi.VOIDP], rffi.INT)
+    if GETTIMEOFDAY_NO_TZ:
+        c_gettimeofday = external('gettimeofday',
+                                  [lltype.Ptr(TIMEVAL)], rffi.INT)
+    else:
+        c_gettimeofday = external('gettimeofday',
+                                  [lltype.Ptr(TIMEVAL), rffi.VOIDP], rffi.INT)
     def gettimeofday(space, w_info=None):
-        with lltype.scoped_alloc(Cconfig.timeval) as timeval:
-            ret = c_gettimeofday(timeval, rffi.NULL)
-            if ret == 0:
-                _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                return space.wrap(timeval.tv_sec + timeval.usec * 1e-6)
-        #TODO: Figure out ftime
+        void = lltype.nullptr(rffi.VOIDP.TO)
+        if False:
+            with lltype.scoped_alloc(Cconfig.timeval) as timeval:
+                if GETTIMEOFDAY_NO_TZ:
+                    errcode = c_gettimeofday(t)
+                else:
+                    errcode = c_gettimeofday(t, void)
+                if rffi.cast(rffi.LONG, errcode) == 0:
+                    _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
+                    return space.wrap(timeval.tv_sec + timeval.usec * 1e-6)
         if HAS_FTIME:
-            pass
+            with lltype.scoped_alloc(TIMEB) as t:
+                c_ftime(t)
+                # The cpython code multiplies by 1000, but rtime.py multiplies
+                # by .001 instead. Why? 
+                # TODO: Get this clarified.
+                result = (float(intmask(t.c_time)) +
+                          float(intmask(t.c_millitm)) * 0.001)
+                if w_info is not None:
+                    _setinfo(space, w_info, "ftime()", 1e-3,
+                             space.w_False, space.w_True) 
+            return space.wrap(result)
         else:
             if w_info:
                 _setinfo(space, w_info, "time()", 1.0, False, True)
@@ -589,7 +610,7 @@
 
     # Can't piggy back on time.time because time.time delegates to the 
     # host python's time.time (so we can't see the internals)
-    if HAS_CLOCK_GETTIME and False:
+    if HAS_CLOCK_GETTIME:
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(cConfig.CLOCK_REALTIME, timespec)
             if ret != 0:
@@ -608,7 +629,7 @@
             secs = _timespec_to_seconds(timespec)
             return secs
     else:
-        return gettimeofday(w_info)
+        return gettimeofday(space, w_info)
 
 def clock(space):
     """clock() -> floating point number

From pypy.commits at gmail.com  Mon Jun 20 18:16:59 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Commit to ask about the error.
Message-ID: <57686b5b.921f1c0a.23e2d.3269@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85281:610233617927
Date: 2016-05-31 10:28 -0400
http://bitbucket.org/pypy/pypy/changeset/610233617927/

Log:	Commit to ask about the error.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -150,6 +150,7 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
+    has_ftime = platform.Has('ftime')
     CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
@@ -182,13 +183,6 @@
         ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
-    # TODO: Figure out how to implement this...
-    CConfig.ULARGE_INTEGER = platform.Struct("struct ULARGE_INTEGER", [
-        ("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
 if _MACOSX:
     CConfig.TIMEBASE_INFO = platform.Struct("struct mach_timebase_info", [
         ("numer", rffi.UINT),
@@ -224,61 +218,11 @@
 
 CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
 HAS_CLOCK_GETTIME = cConfig.has_clock_gettime
+HAS_FTIME = cConfig.has_ftime
 clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday',
-                              [cConfig.timeval, rffi.VOIDP], rffi.INT)
-    if _WIN:
-        GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime',
-                                          [rwin32.FILETIME],
-                                           lltype.VOID)
-        LPDWORD = rwin32.LPDWORD
-        _GetSystemTimeAdjustment = rwin32.winexternal(
-                                                'GetSystemTimeAdjustment',
-                                                [LPDWORD, LPDWORD, rwin32.LPBOOL], 
-                                                rffi.INT)
-        def gettimeofday(space, w_info=None):
-            with lltype.scoped_alloc(rwin32.FILETIME) as system_time, \
-                 lltype.scoped_alloc(CConfig.ULARGE_INTEGER) as large, \
-                 lltype.scoped_alloc(rffi.ULONGLONG) as microseconds:
-
-                GetSystemTimeAsFileTime(system_time)
-                large.u.LowPart = system_time.dwLowDateTime
-                large.u.HighPart = system_time.dwHighDateTime
-                microseconds = system_time.Quadpart / (10 - 11644473600000000)
-                tp.tv_sec = microseconds / 1000000
-                tp.tv_usec = microseconds % 1000000
-                if w_info:
-                    with lltype.scoped_alloc(rwin32.DWORD) as time_adjustment, \
-                         lltype.scoped_alloc(rwin32.DWORD) as time_increment, \
-                         lltype.scoped_alloc(rwin32.BOOL) as is_time_adjustmentDisabled:
-                        w_info.implementation = "GetSystemTimeAsFileTime()"
-                        w_info.monotonic = space.w_False
-                        _GetSystemTimeAdjustment(time_adjustment, time_increment,
-                                                is_time_adjustment_disabled)
-                        w_info.resolution = time_increment * 1e-7
-                        w_info.adjustable = space.w_True
-            # TODO: Find docs for ftime
-            return space.wrap(1)
-
-    else:
-        def gettimeofday(space, w_info=None):
-            with lltype.scoped_alloc(CConfig.timeval) as timeval:
-                ret = c_gettimeofday(timeval, rffi.NULL)
-                if ret != 0:
-                    raise exception_from_saved_errno(space, space.w_OSError)
-                
-                space.setattr(w_info, space.wrap("implementation"),
-                              space.wrap("gettimeofday()"))
-                space.setattr(w_info, space.wrap("resolution"), 1e-6)
-                space.setattr(w_info, space.wrap("monotonic"), space.w_False)
-                space.setattr(w_info, space.wrap("adjustable"), space.w_True)
-
-                seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
-            return space.wrap(seconds)
 
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
@@ -287,6 +231,56 @@
 c_mktime = external('mktime', [TM_P], rffi.TIME_T)
 c_localtime = external('localtime', [rffi.TIME_TP], TM_P,
                        save_err=rffi.RFFI_SAVE_ERRNO)
+
+if _WIN:
+    GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime',
+                                      [rwin32.FILETIME],
+                                       rffi.VOIDP)
+    LPDWORD = rwin32.LPDWORD
+    _GetSystemTimeAdjustment = rwin32.winexternal(
+                                            'GetSystemTimeAdjustment',
+                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
+                                            rffi.INT)
+    def gettimeofday(space, w_info=None):
+        with lltype.scoped_alloc(rwin32.FILETIME) as system_time:#, \
+             #lltype.scoped_alloc(rffi.ULONGLONG) as microseconds:
+
+            GetSystemTimeAsFileTime(system_time)
+            quad_part = (system_time.c_dwLowDateTime |
+                         system_time.c_dwHighDateTime << 32)
+            microseconds = quad_part / 10 - 11644473600000000
+            tv_sec = microseconds / 1000000
+            tv_usec = microseconds % 1000000
+            if w_info:
+                with lltype.scoped_alloc(rwin32.DWORD) as time_adjustment, \
+                     lltype.scoped_alloc(rwin32.DWORD) as time_increment, \
+                     lltype.scoped_alloc(rwin32.BOOL) as is_time_adjustment_disabled:
+                    w_info.implementation = "GetSystemTimeAsFileTime()"
+                    w_info.monotonic = space.w_False
+                    _GetSystemTimeAdjustment(time_adjustment, time_increment,
+                                             is_time_adjustment_disabled)
+                    w_info.resolution = time_increment * 1e-7
+                    w_info.adjustable = space.w_True
+                    # TODO: Check that the 'casting' is correct here
+                    return space.wrap(tv_sec + tv_usec * 1e-6)
+
+else:
+    c_gettimeofday = external('gettimeofday',
+                              [cConfig.timeval, rffi.VOIDP], rffi.INT)
+    def gettimeofday(space, w_info=None):
+        with lltype.scoped_alloc(Cconfig.timeval) as timeval:
+            ret = c_gettimeofday(timeval, rffi.NULL)
+            if ret == 0:
+                _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
+                return space.wrap(timeval.tv_sec + timeval.usec * 1e-6)
+        #TODO: Figure out ftime
+        if HAS_FTIME:
+            pass
+        else:
+            if w_info:
+                _setinfo(space, w_info, "time()", 1.0, False, True)
+            return space.wrap(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
+
 if HAS_CLOCK_GETTIME:
     from rpython.rlib.rtime import TIMESPEC, c_clock_gettime
     c_clock_settime = external('clock_settime',
@@ -587,16 +581,12 @@
     if not 0 <= rffi.getintfield(t_ref, 'c_tm_yday') <= 365:
         raise oefmt(space.w_ValueError, "day of year out of range")
 
-def time(space):
+def time(space, w_info=None):
     """time() -> floating point number
 
     Return the current time in seconds since the Epoch.
     Fractions of a second may be present if the system clock provides them."""
 
-    secs = pytime.time()
-    return space.wrap(secs)
-
-def get_time_time_clock_info(space, w_info):
     # Can't piggy back on time.time because time.time delegates to the 
     # host python's time.time (so we can't see the internals)
     if HAS_CLOCK_GETTIME and False:
@@ -828,15 +818,12 @@
 
 
 if _WIN:
-    # untested so far
     _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
     def monotonic(space, w_info=None):
         result = 0
         if HAS_GETTICKCOUNT64:
-            print('has count64'.encode('ascii'))
             result = _GetTickCount64() * 1e-3
         else:
-            print("nocount64")
             ticks = _GetTickCount()
             if ticks < time_state.last_ticks:
                 time_state.n_overflow += 1
@@ -851,11 +838,9 @@
             else:
                 implementation = "GetTickCount()"
             resolution = 1e-7
-            print("creating a thing".encode("ascii"))
             with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_adjustment, \
                  lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_increment, \
                  lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
-                print("CREATED".encode("ascii"))
                 ok = _GetSystemTimeAdjustment(time_adjustment,
                                               time_increment,
                                               is_time_adjustment_disabled)
@@ -864,11 +849,11 @@
                     raise wrap_windowserror(space,
                         rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
                 resolution = resolution * time_increment[0]
-            print("out of with".encode("ascii"))
             _setinfo(space, w_info, implementation, resolution, True, False)
         return space.wrap(result)
 
 elif _MACOSX:
+    # Completely untested afaik
     c_mach_timebase_info = external('mach_timebase_info',
                                     [lltype.Ptr(cConfig.TIMEBASE_INFO)],
                                     lltype.Void)
@@ -903,6 +888,9 @@
         else:
             clk_id = cConfig.CLOCK_MONOTONIC
             implementation = "clock_gettime(CLOCK_MONOTONIC)"
+        # Will the exception bubble up appropriately or do we
+        # need to add an explicit try/except block to reraise it?
+        # TODO: Ask on irc
         w_result = clock_gettime(space, clk_id)
         if w_info is not None:
             with lltype.scoped_alloc(TIMESPEC) as tsres:
@@ -1004,7 +992,7 @@
         Return the CPU time or real time since the start of the process or since
         the first call to clock().  This has as much precision as the system
         records."""
-        return space.wrap(win_perf_counter(space, w_info=w_info))
+        return space.wrap(perf_counter(space, w_info=w_info))
 
 else:
     _clock = external('clock', [], clock_t)

From pypy.commits at gmail.com  Mon Jun 20 18:16:42 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge with upstream.
Message-ID: <57686b4a.2457c20a.ec0a.ffffaa24@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85272:abc95a8adea5
Date: 2016-05-24 13:04 -0400
http://bitbucket.org/pypy/pypy/changeset/abc95a8adea5/

Log:	Merge with upstream.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -18,6 +18,8 @@
     "exceptions", "_io", "sys", "builtins", "posix", "_warnings",
     "itertools", "_frozen_importlib",
 ])
+if sys.platform == "win32":
+    essential_modules.add("_winreg")
 
 default_modules = essential_modules.copy()
 default_modules.update([
@@ -60,7 +62,6 @@
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
-    working_modules.add("_winreg")
     # unix only modules
     for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses",
                  "_posixsubprocess"]:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -593,9 +593,6 @@
         # lives in pypy/module/exceptions, we rename it below for
         # sys.builtin_module_names
         bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
-        if sys.platform.startswith("win"):
-            self.setbuiltinmodule('_winreg')
-            bootstrap_modules.add('_winreg')
         installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
diff --git a/pypy/module/imp/test/support.py b/pypy/module/imp/test/support.py
--- a/pypy/module/imp/test/support.py
+++ b/pypy/module/imp/test/support.py
@@ -4,8 +4,10 @@
 
     def setup_class(cls):
         space = cls.space
-        cls.w_testfn_unencodable = space.wrap(get_unencodable())
-        cls.w_special_char = space.wrap(get_special_char())
+        cls.testfn_unencodable = get_unencodable()
+        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
+        cls.special_char = get_special_char()
+        cls.w_special_char = space.wrap(cls.special_char)
 
 def get_unencodable():
     """Copy of the stdlib's support.TESTFN_UNENCODABLE:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -133,10 +133,9 @@
              line2 = "# encoding: iso-8859-1\n",
              bad = "# encoding: uft-8\n")
 
-    w_special_char = getattr(cls, 'w_special_char', None)
-    if not space.is_none(w_special_char):
-        special_char = space.unicode_w(w_special_char).encode(
-            sys.getfilesystemencoding())
+    special_char = cls.special_char
+    if special_char is not None:
+        special_char = special_char.encode(sys.getfilesystemencoding())
         p.join(special_char + '.py').write('pass')
 
     # create a .pyw file
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -166,7 +166,8 @@
 def path_or_fd(allow_fd=True):
     return _PathOrFd if allow_fd else _JustPath
 
-DEFAULT_DIR_FD = getattr(rposix, 'AT_FDCWD', -100)
+_HAVE_AT_FDCWD = getattr(rposix, 'AT_FDCWD', None) is not None
+DEFAULT_DIR_FD = rposix.AT_FDCWD if _HAVE_AT_FDCWD else -100
 DIR_FD_AVAILABLE = False
 
 @specialize.arg(2)
@@ -222,11 +223,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
-        else:
+        if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             fd = rposix.openat(path, flags, mode, dir_fd)
+        else:
+            fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     return space.wrap(fd)
@@ -555,7 +556,7 @@
     dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=kwonly(bool),
     follow_symlinks=kwonly(bool))
 def access(space, w_path, mode,
-        dir_fd=DEFAULT_DIR_FD, effective_ids=True, follow_symlinks=True):
+        dir_fd=DEFAULT_DIR_FD, effective_ids=False, follow_symlinks=True):
     """\
 access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
 
@@ -585,12 +586,14 @@
             raise argument_unavailable(space, "access", "effective_ids")
 
     try:
-        if dir_fd == DEFAULT_DIR_FD and follow_symlinks and not effective_ids:
-            ok = dispatch_filename(rposix.access)(space, w_path, mode)
-        else:
+        if (rposix.HAVE_FACCESSAT and
+            (dir_fd != DEFAULT_DIR_FD or not follow_symlinks or
+             effective_ids)):
             path = space.fsencode_w(w_path)
             ok = rposix.faccessat(path, mode,
                 dir_fd, effective_ids, follow_symlinks)
+        else:
+            ok = dispatch_filename(rposix.access)(space, w_path, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -635,11 +638,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.unlink)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=False)
+        else:
+            dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -654,11 +657,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.unlink)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=False)
+        else:
+            dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -721,11 +724,11 @@
 
 The mode argument is ignored on Windows."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.mkdir)(space, w_path, mode)
-        else:
+        if rposix.HAVE_MKDIRAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.mkdirat(path, mode, dir_fd)
+        else:
+            dispatch_filename(rposix.mkdir)(space, w_path, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -740,11 +743,11 @@
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
     try:
-        if dir_fd == DEFAULT_DIR_FD:
-            dispatch_filename(rposix.rmdir)(space, w_path)
-        else:
+        if rposix.HAVE_UNLINKAT and dir_fd != DEFAULT_DIR_FD:
             path = space.fsencode_w(w_path)
             rposix.unlinkat(path, dir_fd, removedir=True)
+        else:
+            dispatch_filename(rposix.rmdir)(space, w_path)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
 
@@ -976,7 +979,8 @@
 src_dir_fd and dst_dir_fd, may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD):
+        if (rposix.HAVE_RENAMEAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD)):
             src = space.fsencode_w(w_src)
             dst = space.fsencode_w(w_dst)
             rposix.renameat(src, dst, src_dir_fd, dst_dir_fd)
@@ -999,7 +1003,8 @@
 src_dir_fd and dst_dir_fd, may not be implemented on your platform.
   If they are unavailable, using them will raise a NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD):
+        if (rposix.HAVE_RENAMEAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD)):
             src = space.fsencode_w(w_src)
             dst = space.fsencode_w(w_dst)
             rposix.renameat(src, dst, src_dir_fd, dst_dir_fd)
@@ -1110,8 +1115,9 @@
   platform.  If they are unavailable, using them will raise a
   NotImplementedError."""
     try:
-        if (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD
-                or not follow_symlinks):
+        if (rposix.HAVE_LINKAT and
+            (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD
+             or not follow_symlinks)):
             rposix.linkat(src, dst, src_dir_fd, dst_dir_fd, follow_symlinks)
         else:
             rposix.link(src, dst)
@@ -1442,31 +1448,32 @@
             # see comment above
             raise wrap_oserror(space, e)
 
+    if (rposix.HAVE_LUTIMES and
+        (dir_fd == DEFAULT_DIR_FD and not follow_symlinks)):
+        path_b = path.as_bytes
+        if path_b is None:
+            raise oefmt(space.w_NotImplementedError,
+                        "utime: unsupported value for 'path'")
+        try:
+            if now:
+                rposix.lutimes(path_b, None)
+            else:
+                rposix.lutimes(path_b, (atime_s, atime_ns))
+            return
+        except OSError as e:
+            # see comment above
+            raise wrap_oserror(space, e)
+
+    # XXX: missing utime_dir_fd support
+
     if not follow_symlinks:
         raise argument_unavailable(space, "utime", "follow_symlinks")
 
-    if not space.is_w(w_ns, space.w_None):
-        raise oefmt(space.w_NotImplementedError,
-            "utime: 'ns' unsupported on this platform on PyPy")
-    if now:
-        try:
+    try:
+        if now:
             call_rposix(utime_now, path, None)
-        except OSError as e:
-            # see comment above
-            raise wrap_oserror(space, e)
-    try:
-        msg = "utime() arg 2 must be a tuple (atime, mtime) or None"
-        args_w = space.fixedview(w_times)
-        if len(args_w) != 2:
-            raise oefmt(space.w_TypeError, msg)
-        actime = space.float_w(args_w[0], allow_conversion=False)
-        modtime = space.float_w(args_w[1], allow_conversion=False)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        raise oefmt(space.w_TypeError, msg)
-    try:
-        call_rposix(rposix.utime, path, (actime, modtime))
+        else:
+            call_rposix(rposix.utime, path, (atime_s, mtime_s))
     except OSError as e:
         # see comment above
         raise wrap_oserror(space, e)
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -1,4 +1,4 @@
-import os, random, struct
+import sys, os, random, struct
 import py
 from rpython.jit.backend.x86 import rx86
 from rpython.rlib.rarithmetic import intmask
@@ -257,6 +257,9 @@
         g.close()
         error = [line for line in got.splitlines() if 'error' in line.lower()]
         if error:
+            if (sys.maxint <= 2**32 and
+                    'no compiled in support for x86_64' in error[0]):
+                py.test.skip(error)
             raise Exception("Assembler got an error: %r" % error[0])
         error = [line for line in got.splitlines()
                  if 'warning' in line.lower()]
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1219,21 +1219,14 @@
         if times is None:
             error = c_utime(path, lltype.nullptr(UTIMBUFP.TO))
         else:
-            actime, modtime = times
             if HAVE_UTIMES:
-                import math
-                l_times = lltype.malloc(TIMEVAL2P.TO, 2, flavor='raw')
-                fracpart, intpart = math.modf(actime)
-                rffi.setintfield(l_times[0], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[0], 'c_tv_usec', int(fracpart * 1e6))
-                fracpart, intpart = math.modf(modtime)
-                rffi.setintfield(l_times[1], 'c_tv_sec', int(intpart))
-                rffi.setintfield(l_times[1], 'c_tv_usec', int(fracpart * 1e6))
-                error = c_utimes(path, l_times)
-                lltype.free(l_times, flavor='raw')
+                with lltype.scoped_alloc(TIMEVAL2P.TO, 2) as l_timeval2p:
+                    times_to_timeval2p(times, l_timeval2p)
+                    error = c_utimes(path, l_timeval2p)
             else:
                 # we only have utime(), which does not allow
                 # sub-second resolution
+                actime, modtime = times
                 l_utimbuf = lltype.malloc(UTIMBUFP.TO, flavor='raw')
                 l_utimbuf.c_actime  = rffi.r_time_t(actime)
                 l_utimbuf.c_modtime = rffi.r_time_t(modtime)
@@ -1276,6 +1269,17 @@
             lltype.free(atime, flavor='raw')
             lltype.free(mtime, flavor='raw')
 
+def times_to_timeval2p(times, l_timeval2p):
+    actime, modtime = times
+    _time_to_timeval(actime, l_timeval2p[0])
+    _time_to_timeval(modtime, l_timeval2p[1])
+
+def _time_to_timeval(t, l_timeval):
+    import math
+    fracpart, intpart = math.modf(t)
+    rffi.setintfield(l_timeval, 'c_tv_sec', int(intpart))
+    rffi.setintfield(l_timeval, 'c_tv_usec', int(fracpart * 1e6))
+
 if not _WIN32:
     TMSP = lltype.Ptr(TMS)
     c_times = external('times', [TMSP], CLOCK_T,
@@ -1763,6 +1767,7 @@
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
         includes=['sys/stat.h',
+                  'sys/time.h',
                   'unistd.h',
                   'fcntl.h'],
     )
@@ -1918,6 +1923,21 @@
         lltype.free(l_times, flavor='raw')
         handle_posix_error('utimensat', error)
 
+if HAVE_LUTIMES:
+    c_lutimes = external('lutimes',
+        [rffi.CCHARP, TIMEVAL2P], rffi.INT,
+        save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @specialize.argtype(1)
+    def lutimes(pathname, times):
+        if times is None:
+            error = c_lutimes(pathname, lltype.nullptr(TIMEVAL2P.TO))
+        else:
+            with lltype.scoped_alloc(TIMEVAL2P.TO, 2) as l_timeval2p:
+                times_to_timeval2p(times, l_timeval2p)
+                error = c_lutimes(pathname, l_timeval2p)
+        handle_posix_error('lutimes', error)
+
 if HAVE_MKDIRAT:
     c_mkdirat = external('mkdirat',
         [rffi.INT, rffi.CCHARP, rffi.INT], rffi.INT,

From pypy.commits at gmail.com  Mon Jun 20 18:17:03 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:17:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merging with upstream.
Message-ID: <57686b5f.949a1c0a.db1f3.3544@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85283:3509fa74996d
Date: 2016-06-03 01:00 -0400
http://bitbucket.org/pypy/pypy/changeset/3509fa74996d/

Log:	Merging with upstream.

diff too long, truncating to 2000 out of 5949 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -23,3 +23,5 @@
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat 
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -43,11 +43,13 @@
         return short
     return text
 
-def run_multiline_interactive_console(mainmodule=None):
+def run_multiline_interactive_console(mainmodule=None, future_flags=0):
     import code
     import __main__
     mainmodule = mainmodule or __main__
     console = code.InteractiveConsole(mainmodule.__dict__, filename='')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -70,9 +70,6 @@
 bz2
     libbz2
 
-lzma (PyPy3 only)
-    liblzma
-
 pyexpat
     libexpat1
 
@@ -98,11 +95,16 @@
 tk
     tk-dev
 
+lzma (PyPy3 only)
+    liblzma
+
+To run untranslated tests, you need the Boehm garbage collector libgc.
+
 On Debian, this is the command to install all build-time dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev
+    tk-dev libgc-dev liblzma-dev
 
 For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
 
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -49,6 +49,13 @@
    release-0.6
 
 
+CPython 3.3 compatible versions
+-------------------------------
+
+.. toctree::
+
+   release-pypy3.3-v5.2-alpha1.rst
+
 CPython 3.2 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/release-pypy3.3-v5.2-alpha1.rst b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy3.3-v5.2-alpha1.rst
@@ -0,0 +1,69 @@
+===================
+PyPy3 v5.2 alpha 1
+===================
+
+We're pleased to announce the first alpha release of PyPy3.3 v5.2. This is the
+first release of PyPy which targets Python 3.3 (3.3.5) compatibility.
+
+We would like to thank all of the people who donated_ to the `py3k proposal`_
+for supporting the work that went into this and future releases.
+
+You can download the PyPy3.3 v5.2 alpha 1 release here:
+
+    http://pypy.org/download.html#python-3-3-5-compatible-pypy3-3-v5-2
+
+Highlights
+==========
+
+* Python 3.3.5 support!
+
+  - Being an early alpha release, there are some `missing features`_ such as a
+    `PEP 393-like space efficient string representation`_ and `known issues`_
+    including performance regressions (e.g. issue `#2305`_). The focus for this
+    release has been updating to 3.3 compatibility. Windows is also not yet
+    supported.
+
+* `ensurepip`_ is also included (it's only included in CPython 3 >= 3.4).
+
+What is PyPy?
+==============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7.10 and one day 3.3.5. It's fast due to its integrated tracing JIT
+compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems except Windows
+    (Linux 32/64, Mac OS X 64, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+Please try it out and let us know what you think. We welcome feedback, we know
+you are using PyPy, please tell us about it!
+
+We'd especially like to thank these people for their contributions to this
+release:
+
+Manuel Jacob, Ronan Lamy, Mark Young, Amaury Forgeot d'Arc, Philip Jenvey,
+Martin Matusiak, Vasily Kuznetsov, Matti Picus, Armin Rigo and many others.
+
+Cheers
+
+The PyPy Team
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`PEP 393-like space efficient string representation`: https://bitbucket.org/pypy/pypy/issues/2309/optimized-unicode-representation
+.. _`missing features`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3+%28running+Python+3.x%29&kind=enhancement
+.. _`known issues`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open&component=PyPy3%20%28running%20Python%203.x%29
+.. _`#2305`: https://bitbucket.org/pypy/pypy/issues/2305
+.. _`ensurepip`: https://docs.python.org/3/library/ensurepip.html#module-ensurepip
+.. _`dynamic languages`: http://pypyjs.org
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -105,3 +105,29 @@
 Fix some warnings when compiling CPython C extension modules
 
 .. branch: syntax_fix
+
+.. branch: remove-raisingops
+
+Remove most of the _ovf, _zer and _val operations from RPython.  Kills
+quite some code internally, and allows the JIT to do better
+optimizations: for example, app-level code like ``x / 2`` or ``x % 2``
+can now be turned into ``x >> 1`` or ``x & 1``, even if x is possibly
+negative.
+
+.. branch: cpyext-old-buffers
+
+Generalize cpyext old-style buffers to more than just str/buffer, add support for mmap
+
+.. branch: numpy-includes
+
+Move _numpypy headers into a directory so they are not picked up by upstream numpy, scipy
+This allows building upstream numpy and scipy in pypy via cpyext
+
+.. branch: traceviewer-common-merge-point-formats
+
+Teach RPython JIT's off-line traceviewer the most common ``debug_merge_point`` formats.
+
+.. branch: cpyext-pickle
+
+Enable pickling of W_PyCFunctionObject by monkeypatching pickle.Pickler.dispatch
+at cpyext import time
diff --git a/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.1.1-alpha1.rst
@@ -0,0 +1,10 @@
+=================================
+What's new in PyPy3 5.1.1 alpha 1
+=================================
+
+.. A recent revision, ignoring all other branches for this release
+.. startrev: 29d14733e007
+
+.. branch: py3.3
+
+Python 3.3 compatibility
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -238,6 +238,15 @@
 for use. The release packaging script will pick up the tcltk runtime in the lib
 directory and put it in the archive.
 
+The lzma compression library
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.3 ship with CFFI wrappers for the lzma library, which can be
+downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
+5.0.5, a prebuilt version can be downloaded from
+http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
+http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
+
 
 Using the mingw compiler
 ------------------------
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -72,6 +72,11 @@
         print('Goodbye2')  # should not be reached
     """)
 
+script_with_future = getscript("""
+    from __future__ import division
+    from __future__ import print_function
+    """)
+
 
 @contextmanager
 def setpythonpath():
@@ -441,6 +446,31 @@
         finally:
             os.environ['PYTHONSTARTUP'] = old
 
+    def test_future_in_executed_script(self):
+        child = self.spawn(['-i', script_with_future])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_python_startup(self, monkeypatch):
+        monkeypatch.setenv('PYTHONSTARTUP', script_with_future)
+        child = self.spawn([])
+        child.expect('>>> ')
+        child.sendline('x=1; print(x/2, 3/4)')
+        child.expect('0.5 0.75')
+
+    def test_future_in_cmd(self):
+        child = self.spawn(['-i', '-c', 'from __future__ import division'])
+        child.expect('>>> ')
+        child.sendline('x=1; x/2; 3/4')
+        child.expect('0.5')
+        child.expect('0.75')
+
+    def test_cmd_co_name(self):
+        child = self.spawn(['-c',
+                    'import sys; print sys._getframe(0).f_code.co_name'])
+        child.expect('')
+
     def test_ignore_python_inspect(self):
         os.environ['PYTHONINSPECT_'] = '1'
         try:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -12,7 +12,8 @@
 
 
 class TypeDef(object):
-    def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict):
+    def __init__(self, __name, __base=None, __total_ordering__=None,
+                 __buffer=None, **rawdict):
         "NOT_RPYTHON: initialization-time only"
         self.name = __name
         if __base is None:
@@ -22,6 +23,8 @@
         else:
             bases = [__base]
         self.bases = bases
+        assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+        self.buffer = __buffer
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
         # no __del__: use an RPython _finalize_() method and register_finalizer
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -2,6 +2,19 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import jit
+
+
+# XXX maybe temporary: hide llop.int_{floordiv,mod} from the JIT,
+#     because now it expects only Python-style divisions, not the
+#     C-style divisions of these two ll operations
+ at jit.dont_look_inside
+def _int_floordiv(n, m):
+    return llop.int_floordiv(lltype.Signed, n, m)
+
+ at jit.dont_look_inside
+def _int_mod(n, m):
+    return llop.int_mod(lltype.Signed, n, m)
 
 
 @unwrap_spec(n=int, m=int)
@@ -18,11 +31,11 @@
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(llop.int_floordiv(lltype.Signed, n, m))
+    return space.wrap(_int_floordiv(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(llop.int_mod(lltype.Signed, n, m))
+    return space.wrap(_int_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1784,3 +1784,9 @@
         assert ffi.list_types() == (['CFFIb', 'CFFIbb', 'CFFIbbb'],
                                     ['CFFIa', 'CFFIcc', 'CFFIccc'],
                                     ['CFFIaa', 'CFFIaaa', 'CFFIg'])
+
+    def test_FFIFunctionWrapper(self):
+        ffi, lib = self.prepare("void f(void);", "test_FFIFunctionWrapper",
+                                "void f(void) { }")
+        assert lib.f.__get__(42) is lib.f
+        assert lib.f.__get__(42, int) is lib.f
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -100,6 +100,11 @@
         doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename)
         return space.wrap(doc)
 
+    def descr_get(self, space, w_obj, w_type=None):
+        # never bind anything, but a __get__ is still present so that
+        # pydoc displays useful information (namely, the __repr__)
+        return self
+
 
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
@@ -136,5 +141,6 @@
         __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper),
         __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper),
         __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc),
+        __get__ = interp2app(W_FunctionWrapper.descr_get),
         )
 W_FunctionWrapper.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from pypy.interpreter import gateway
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
 
@@ -14,6 +15,12 @@
 
     def startup(self, space):
         space.fromcache(State).startup(space)
+        method = pypy.module.cpyext.typeobject.get_new_method_def(space)
+        w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
+        space.appexec([space.type(w_obj)], """(methodtype):
+            from pickle import Pickler 
+            Pickler.dispatch[methodtype] = Pickler.save_global
+        """)
 
     def register_atexit(self, function):
         if len(self.atexit_funcs) >= 32:
@@ -64,6 +71,7 @@
 import pypy.module.cpyext.pyfile
 import pypy.module.cpyext.pystrtod
 import pypy.module.cpyext.pytraceback
+import pypy.module.cpyext.methodobject
 
 # now that all rffi_platform.Struct types are registered, configure them
 api.configure_types()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -161,12 +161,13 @@
 
     if copy_numpy_headers:
         try:
-            dstdir.mkdir('numpy')
+            dstdir.mkdir('_numpypy')
+            dstdir.mkdir('_numpypy/numpy')
         except py.error.EEXIST:
             pass
-        numpy_dstdir = dstdir / 'numpy'
+        numpy_dstdir = dstdir / '_numpypy' / 'numpy'
 
-        numpy_include_dir = include_dir / 'numpy'
+        numpy_include_dir = include_dir / '_numpypy' / 'numpy'
         numpy_headers = numpy_include_dir.listdir('*.h') + numpy_include_dir.listdir('*.inl')
         _copy_header_files(numpy_headers, numpy_dstdir)
 
diff --git a/pypy/module/cpyext/include/numpy/README b/pypy/module/cpyext/include/_numpypy/numpy/README
rename from pypy/module/cpyext/include/numpy/README
rename to pypy/module/cpyext/include/_numpypy/numpy/README
diff --git a/pypy/module/cpyext/include/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
rename from pypy/module/cpyext/include/numpy/__multiarray_api.h
rename to pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h
diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/_numpypy/numpy/arrayobject.h
rename from pypy/module/cpyext/include/numpy/arrayobject.h
rename to pypy/module/cpyext/include/_numpypy/numpy/arrayobject.h
diff --git a/pypy/module/cpyext/include/numpy/ndarraytypes.h b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
rename from pypy/module/cpyext/include/numpy/ndarraytypes.h
rename to pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
rename from pypy/module/cpyext/include/numpy/npy_3kcompat.h
rename to pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
diff --git a/pypy/module/cpyext/include/numpy/npy_common.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h
rename from pypy/module/cpyext/include/numpy/npy_common.h
rename to pypy/module/cpyext/include/_numpypy/numpy/npy_common.h
diff --git a/pypy/module/cpyext/include/numpy/old_defines.h b/pypy/module/cpyext/include/_numpypy/numpy/old_defines.h
rename from pypy/module/cpyext/include/numpy/old_defines.h
rename to pypy/module/cpyext/include/_numpypy/numpy/old_defines.h
diff --git a/pypy/module/cpyext/include/cStringIO.h b/pypy/module/cpyext/include/cStringIO.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/cStringIO.h
@@ -0,0 +1,73 @@
+#ifndef Py_CSTRINGIO_H
+#define Py_CSTRINGIO_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+
+  This header provides access to cStringIO objects from C.
+  Functions are provided for calling cStringIO objects and
+  macros are provided for testing whether you have cStringIO
+  objects.
+
+  Before calling any of the functions or macros, you must initialize
+  the routines with:
+
+    PycString_IMPORT
+
+  This would typically be done in your init function.
+
+*/
+
+#define PycStringIO_CAPSULE_NAME "cStringIO.cStringIO_CAPI"
+
+#define PycString_IMPORT \
+  PycStringIO = ((struct PycStringIO_CAPI*)PyCapsule_Import(\
+    PycStringIO_CAPSULE_NAME, 0))
+
+/* Basic functions to manipulate cStringIO objects from C */
+
+static struct PycStringIO_CAPI {
+
+ /* Read a string from an input object.  If the last argument
+    is -1, the remainder will be read.
+    */
+  int(*cread)(PyObject *, char **, Py_ssize_t);
+
+ /* Read a line from an input object.  Returns the length of the read
+    line as an int and a pointer inside the object buffer as char** (so
+    the caller doesn't have to provide its own buffer as destination).
+    */
+  int(*creadline)(PyObject *, char **);
+
+  /* Write a string to an output object*/
+  int(*cwrite)(PyObject *, const char *, Py_ssize_t);
+
+  /* Get the output object as a Python string (returns new reference). */
+  PyObject *(*cgetvalue)(PyObject *);
+
+  /* Create a new output object */
+  PyObject *(*NewOutput)(int);
+
+  /* Create an input object from a Python string
+     (copies the Python string reference).
+     */
+  PyObject *(*NewInput)(PyObject *);
+
+  /* The Python types for cStringIO input and output objects.
+     Note that you can do input on an output object.
+     */
+  PyTypeObject *InputType, *OutputType;
+
+} *PycStringIO;
+
+/* These can be used to test if you have one */
+#define PycStringIO_InputCheck(O) \
+  (0) /* Py_TYPE(O)==PycStringIO->InputType) */
+#define PycStringIO_OutputCheck(O) \
+  (0) /* Py_TYPE(O)==PycStringIO->OutputType) */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CSTRINGIO_H */
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -44,8 +44,8 @@
                    dealloc=cfunction_dealloc)
 
 def cfunction_attach(space, py_obj, w_obj):
+    assert isinstance(w_obj, W_PyCFunctionObject)
     py_func = rffi.cast(PyCFunctionObject, py_obj)
-    assert isinstance(w_obj, W_PyCFunctionObject)
     py_func.c_m_ml = w_obj.ml
     py_func.c_m_self = make_ref(space, w_obj.w_self)
     py_func.c_m_module = make_ref(space, w_obj.w_module)
diff --git a/pypy/module/cpyext/src/ndarrayobject.c b/pypy/module/cpyext/src/ndarrayobject.c
--- a/pypy/module/cpyext/src/ndarrayobject.c
+++ b/pypy/module/cpyext/src/ndarrayobject.c
@@ -1,7 +1,7 @@
 
 #include "Python.h"
 #include "pypy_numpy.h"
-#include "numpy/arrayobject.h"
+#include "_numpypy/numpy/arrayobject.h"
 #include    /* memset, memcpy */
 
 void 
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -252,12 +252,16 @@
         lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
 
         w_text = space.wrapbytes("text")
-        assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0
+        ref = make_ref(space, w_text)
+        prev_refcnt = ref.c_ob_refcnt
+        assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0
+        assert ref.c_ob_refcnt == prev_refcnt
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
 
         lltype.free(bufp, flavor='raw')
         lltype.free(lenp, flavor='raw')
+        api.Py_DecRef(ref)
 
     def test_eq(self, space, api):
         assert 1 == api._PyBytes_Eq(space.wrapbytes("hello"), space.wrapbytes("hello"))
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -1,4 +1,5 @@
 import py
+import os
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -238,8 +239,10 @@
             except:
                 skip('numpy not importable')
         else:
-            cls.w_numpy_include = cls.space.wrap([])
-
+            numpy_incl = os.path.abspath(os.path.dirname(__file__) + 
+                                         '/../include/_numpypy')
+            assert os.path.exists(numpy_incl)
+            cls.w_numpy_include = cls.space.wrap([numpy_incl])
 
     def test_ndarray_object_c(self):
         mod = self.import_extension('foo', [
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -125,7 +125,8 @@
         return None
 
     def issubtype_w(self, w_sub, w_type):
-        return w_sub is w_type
+        is_root(w_type)
+        return NonConstant(True)
 
     def isinstance_w(self, w_obj, w_tp):
         try:
@@ -414,6 +415,10 @@
     def warn(self, w_msg, w_warn_type):
         pass
 
+def is_root(w_obj):
+    assert isinstance(w_obj, W_Root)
+is_root.expecting = W_Root
+
 class FloatObject(W_Root):
     tp = FakeSpace.w_float
     def __init__(self, floatval):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -262,7 +262,7 @@
             [i0]
             i1 = int_add(i0, 1)
             i2 = int_sub(i1, 10)
-            i3 = int_floordiv(i2, 100)
+            i3 = int_xor(i2, 100)
             i4 = int_mul(i1, 1000)
             jump(i4)
         """
@@ -298,7 +298,7 @@
             [i0]
             i1 = int_add(i0, 1)
             i2 = int_sub(i1, 10)
-            i3 = int_floordiv(i2, 100)
+            i3 = int_xor(i2, 100)
             i4 = int_mul(i1, 1000)
             jump(i4)
         """
diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py
--- a/pypy/module/pypyjit/test_pypy_c/test_shift.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py
@@ -47,26 +47,74 @@
             res = 0
             a = 0
             while a < 300:
-                assert a >= 0
-                assert 0 <= b <= 10
-                res = a/b     # ID: div
+                res1 = a/b     # ID: div
+                res2 = a/2     # ID: shift
+                res3 = a/11    # ID: mul
+                res += res1 + res2 + res3
                 a += 1
             return res
         #
         log = self.run(main, [3])
-        assert log.result == 99
+        assert log.result == main(3)
         loop, = log.loops_by_filename(self.filepath)
-        if sys.maxint == 2147483647:
-            SHIFT = 31
+        assert loop.match_by_id('div', """
+            i56 = int_eq(i48, %d)
+            i57 = int_and(i56, i37)
+            guard_false(i57, descr=...)
+            i1 = call_i(_, i48, i3, descr=...)
+        """ % (-sys.maxint-1,))
+        assert loop.match_by_id('shift', """
+            i1 = int_rshift(i2, 1)
+        """)
+        if sys.maxint > 2**32:
+            args = (63, -5030930201920786804, 3)
         else:
-            SHIFT = 63
-        assert loop.match_by_id('div', """
-            i10 = int_floordiv(i6, i7)
-            i11 = int_mul(i10, i7)
-            i12 = int_sub(i6, i11)
-            i14 = int_rshift(i12, %d)
-            i15 = int_add(i10, i14)
-        """ % SHIFT)
+            args = (31, -1171354717, 3)
+        assert loop.match_by_id('mul', """
+            i2 = int_rshift(i1, %d)
+            i3 = int_xor(i1, i2)
+            i4 = uint_mul_high(i3, %d)
+            i5 = uint_rshift(i4, %d)
+            i6 = int_xor(i5, i2)
+        """ % args)
+
+    def test_modulo_optimization(self):
+        def main(b):
+            res = 0
+            a = 0
+            while a < 300:
+                res1 = a%b     # ID: mod
+                res2 = a%2     # ID: and
+                res3 = a%11    # ID: mul
+                res += res1 + res2 + res3
+                a += 1
+            return res
+        #
+        log = self.run(main, [3])
+        assert log.result == main(3)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('mod', """
+            i56 = int_eq(i48, %d)
+            i57 = int_and(i56, i37)
+            guard_false(i57, descr=...)
+            i1 = call_i(_, i48, i3, descr=...)
+        """ % (-sys.maxint-1,))
+        assert loop.match_by_id('and', """
+            i1 = int_and(i2, 1)
+        """)
+        if sys.maxint > 2**32:
+            args = (63, -5030930201920786804, 3)
+        else:
+            args = (31, -1171354717, 3)
+        assert loop.match_by_id('mul', """
+            i2 = int_rshift(i1, %d)
+            i3 = int_xor(i1, i2)
+            i4 = uint_mul_high(i3, %d)
+            i5 = uint_rshift(i4, %d)
+            i6 = int_xor(i5, i2)
+            i7 = int_mul(i6, 11)
+            i8 = int_sub(i1, i7)
+        """ % args)
 
     def test_division_to_rshift_allcases(self):
         """
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -1,11 +1,6 @@
 import sys
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
-if sys.maxint == 2147483647:
-    SHIFT = 31
-else:
-    SHIFT = 63
-
 # XXX review the  descrs to replace some EF=5 with EF=4 (elidable)
 
 
@@ -28,10 +23,7 @@
             guard_true(i14, descr=...)
             guard_not_invalidated(descr=...)
             i16 = int_eq(i6, %d)
-            i15 = int_mod(i6, i10)
-            i17 = int_rshift(i15, %d)
-            i18 = int_and(i10, i17)
-            i19 = int_add(i15, i18)
+            i19 = call_i(ConstClass(ll_int_mod__Signed_Signed), i6, i10, descr=)
             i21 = int_lt(i19, 0)
             guard_false(i21, descr=...)
             i22 = int_ge(i19, i10)
@@ -49,7 +41,7 @@
             i34 = int_add(i6, 1)
             --TICK--
             jump(..., descr=...)
-        """ % (-sys.maxint-1, SHIFT))
+        """ % (-sys.maxint-1,))
 
     def test_long(self):
         def main(n):
@@ -62,19 +54,25 @@
         log = self.run(main, [1100], import_site=True)
         assert log.result == main(1100)
         loop, = log.loops_by_filename(self.filepath)
+        if sys.maxint > 2**32:
+            args = (63, -3689348814741910323, 3)
+        else:
+            args = (31, -858993459, 3)
         assert loop.match("""
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
             guard_not_invalidated(descr=...)
             i13 = int_eq(i6, %d)         # value provided below
-            i15 = int_mod(i6, 10)
-            i17 = int_rshift(i15, %d)    # value provided below
-            i18 = int_and(10, i17)
-            i19 = int_add(i15, i18)
-            i21 = int_lt(i19, 0)
-            guard_false(i21, descr=...)
-            i22 = int_ge(i19, 10)
-            guard_false(i22, descr=...)
+
+            # "mod 10" block:
+            i79 = int_rshift(i6, %d)
+            i80 = int_xor(i6, i79)
+            i82 = uint_mul_high(i80, %d)
+            i84 = uint_rshift(i82, %d)
+            i85 = int_xor(i84, i79)
+            i87 = int_mul(i85, 10)
+            i19 = int_sub(i6, i87)
+
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
             strsetitem(p25, 0, i23)
@@ -89,7 +87,7 @@
             guard_no_overflow(descr=...)
             --TICK--
             jump(..., descr=...)
-        """ % (-sys.maxint-1, SHIFT))
+        """ % ((-sys.maxint-1,)+args))
 
     def test_str_mod(self):
         def main(n):
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -115,7 +115,7 @@
 
     if state is not None:    # 'None' for testing only
         lib_extensions = os.path.join(lib_pypy, '__extensions__')
-        state.w_lib_extensions = _w_fsdecode(state.space, lib_extensions)
+        state.w_lib_extensions = state.space.wrap_fsdecoded(lib_extensions)
         importlist.append(lib_extensions)
 
     importlist.append(lib_pypy)
@@ -147,12 +147,12 @@
 
 @unwrap_spec(executable='fsencode')
 def pypy_find_executable(space, executable):
-    return _w_fsdecode(space, find_executable(executable))
+    return space.wrap_fsdecoded(find_executable(executable))
 
 
 @unwrap_spec(filename='fsencode')
 def pypy_resolvedirof(space, filename):
-    return _w_fsdecode(space, resolvedirof(filename))
+    return space.wrap_fsdecoded(resolvedirof(filename))
 
 
 @unwrap_spec(executable='fsencode')
@@ -160,16 +160,12 @@
     path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
         return space.w_None
-    w_prefix = _w_fsdecode(space, prefix)
+    w_prefix = space.wrap_fsdecoded(prefix)
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('base_prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('base_exec_prefix'), w_prefix)
-    return space.newlist([_w_fsdecode(space, p) for p in path])
+    return space.newlist([space.wrap_fsdecoded(p) for p in path])
 
 def pypy_initfsencoding(space):
     space.sys.filesystemencoding = _getfilesystemencoding(space)
-
-
-def _w_fsdecode(space, b):
-    return space.fsdecode(space.wrapbytes(b))
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -7,6 +7,7 @@
 from rpython.rlib import rstring, runicode, rlocale, rfloat, jit
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
+from rpython.rlib.rarithmetic import r_uint, intmask
 
 
 @specialize.argtype(1)
@@ -836,33 +837,37 @@
                 return s
             # This part is slow.
             negative = value < 0
-            value = abs(value)
+            base = r_uint(base)
+            value = r_uint(value)
+            if negative:   # change the sign on the unsigned number: otherwise,
+                value = -value   #   we'd risk overflow if value==-sys.maxint-1
+            #
             buf = ["\0"] * (8 * 8 + 6) # Too much on 32 bit, but who cares?
             i = len(buf) - 1
             while True:
-                div = value // base
-                mod = value - div * base
-                digit = abs(mod)
+                div = value // base         # unsigned
+                mod = value - div * base    # unsigned, always in range(0,base)
+                digit = intmask(mod)
                 digit += ord("0") if digit < 10 else ord("a") - 10
                 buf[i] = chr(digit)
-                value = div
+                value = div                 # unsigned
                 i -= 1
                 if not value:
                     break
-            if base == 2:
+            if base == r_uint(2):
                 buf[i] = "b"
                 buf[i - 1] = "0"
-            elif base == 8:
+            elif base == r_uint(8):
                 buf[i] = "o"
                 buf[i - 1] = "0"
-            elif base == 16:
+            elif base == r_uint(16):
                 buf[i] = "x"
                 buf[i - 1] = "0"
             else:
                 buf[i] = "#"
-                buf[i - 1] = chr(ord("0") + base % 10)
-                if base > 10:
-                    buf[i - 2] = chr(ord("0") + base // 10)
+                buf[i - 1] = chr(ord("0") + intmask(base % r_uint(10)))
+                if base > r_uint(10):
+                    buf[i - 2] = chr(ord("0") + intmask(base // r_uint(10)))
                     i -= 1
             i -= 1
             if negative:
diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py
--- a/pypy/tool/release/force-builds.py
+++ b/pypy/tool/release/force-builds.py
@@ -19,16 +19,16 @@
 BUILDERS = [
     'own-linux-x86-32',
     'own-linux-x86-64',
-    'own-linux-armhf',
+#    'own-linux-armhf',
     'own-win-x86-32',
-    'own-linux-s390x-2',
+    'own-linux-s390x',
 #    'own-macosx-x86-32',
     'pypy-c-jit-linux-x86-32',
     'pypy-c-jit-linux-x86-64',
 #    'pypy-c-jit-freebsd-9-x86-64',
     'pypy-c-jit-macosx-x86-64',
     'pypy-c-jit-win-x86-32',
-    'pypy-c-jit-linux-s390x-2',
+    'pypy-c-jit-linux-s390x',
     'build-pypy-c-jit-linux-armhf-raring',
     'build-pypy-c-jit-linux-armhf-raspbian',
     'build-pypy-c-jit-linux-armel',
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -213,11 +213,6 @@
                    default=False),
         BoolOption("merge_if_blocks", "Merge if ... elif chains",
                    cmdline="--if-block-merge", default=True),
-        BoolOption("raisingop2direct_call",
-                   "Transform operations that can implicitly raise an "
-                   "exception into calls to functions that explicitly "
-                   "raise exceptions",
-                   default=False, cmdline="--raisingop2direct_call"),
         BoolOption("mallocs", "Remove mallocs", default=True),
         BoolOption("constfold", "Constant propagation",
                    default=True),
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -1,6 +1,5 @@
 from rpython.jit.backend.arm import conditions as cond
 from rpython.jit.backend.arm import registers as reg
-from rpython.jit.backend.arm import support
 from rpython.jit.backend.arm.arch import WORD, PC_OFFSET
 from rpython.jit.backend.arm.instruction_builder import define_instructions
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
@@ -17,17 +16,6 @@
     sandboxsafe=True)
 
 
-def binary_helper_call(name):
-    function = getattr(support, 'arm_%s' % name)
-
-    def f(self, c=cond.AL):
-        """Generates a call to a helper function, takes its
-        arguments in r0 and r1, result is placed in r0"""
-        addr = rffi.cast(lltype.Signed, function)
-        self.BL(addr, c)
-    return f
-
-
 class AbstractARMBuilder(object):
     def __init__(self, arch_version=7):
         self.arch_version = arch_version
@@ -348,10 +336,6 @@
         self.write32(c << 28
                     | 0x157ff05f)
 
-    DIV = binary_helper_call('int_div')
-    MOD = binary_helper_call('int_mod')
-    UDIV = binary_helper_call('uint_div')
-
     FMDRR = VMOV_cr     # uh, there are synonyms?
     FMRRD = VMOV_rc
 
diff --git a/rpython/jit/backend/arm/helper/assembler.py b/rpython/jit/backend/arm/helper/assembler.py
--- a/rpython/jit/backend/arm/helper/assembler.py
+++ b/rpython/jit/backend/arm/helper/assembler.py
@@ -46,20 +46,6 @@
     f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_op_by_helper_call(name, opname):
-    helper = getattr(InstrBuilder, opname)
-    def f(self, op, arglocs, regalloc, fcond):
-        assert fcond is not None
-        if op.type != 'v':
-            regs = r.caller_resp[1:] + [r.ip]
-        else:
-            regs = r.caller_resp
-        with saved_registers(self.mc, regs, r.caller_vfp_resp):
-            helper(self.mc, fcond)
-        return fcond
-    f.__name__ = 'emit_op_%s' % name
-    return f
-
 def gen_emit_cmp_op(name, true_cond):
     def f(self, op, arglocs, regalloc, fcond):
         l0, l1, res = arglocs
diff --git a/rpython/jit/backend/arm/helper/regalloc.py b/rpython/jit/backend/arm/helper/regalloc.py
--- a/rpython/jit/backend/arm/helper/regalloc.py
+++ b/rpython/jit/backend/arm/helper/regalloc.py
@@ -72,25 +72,6 @@
     res = self.force_allocate_reg_or_cc(op)
     return [loc1, loc2, res]
 
-def prepare_op_by_helper_call(name):
-    def f(self, op, fcond):
-        assert fcond is not None
-        a0 = op.getarg(0)
-        a1 = op.getarg(1)
-        arg1 = self.rm.make_sure_var_in_reg(a0, selected_reg=r.r0)
-        arg2 = self.rm.make_sure_var_in_reg(a1, selected_reg=r.r1)
-        assert arg1 == r.r0
-        assert arg2 == r.r1
-        if not isinstance(a0, Const) and self.stays_alive(a0):
-            self.force_spill_var(a0)
-        self.possibly_free_vars_for_op(op)
-        self.free_temp_vars()
-        self.after_call(op)
-        self.possibly_free_var(op)
-        return []
-    f.__name__ = name
-    return f
-
 def prepare_int_cmp(self, op, fcond):
     assert fcond is not None
     boxes = list(op.getarglist())
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -3,7 +3,7 @@
 from rpython.jit.backend.arm import registers as r
 from rpython.jit.backend.arm import shift
 from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE
-from rpython.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call,
+from rpython.jit.backend.arm.helper.assembler import (
                                                 gen_emit_op_unary_cmp,
                                                 gen_emit_op_ri,
                                                 gen_emit_cmp_op,
@@ -92,6 +92,11 @@
         self.mc.MUL(res.value, reg1.value, reg2.value)
         return fcond
 
+    def emit_op_uint_mul_high(self, op, arglocs, regalloc, fcond):
+        reg1, reg2, res = arglocs
+        self.mc.UMULL(r.ip.value, res.value, reg1.value, reg2.value)
+        return fcond
+
     def emit_op_int_force_ge_zero(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         self.mc.CMP_ri(arg.value, 0)
@@ -132,10 +137,6 @@
         self.guard_success_cc = c.VC
         return fcond
 
-    emit_op_int_floordiv = gen_emit_op_by_helper_call('int_floordiv', 'DIV')
-    emit_op_int_mod = gen_emit_op_by_helper_call('int_mod', 'MOD')
-    emit_op_uint_floordiv = gen_emit_op_by_helper_call('uint_floordiv', 'UDIV')
-
     emit_op_int_and = gen_emit_op_ri('int_and', 'AND')
     emit_op_int_or = gen_emit_op_ri('int_or', 'ORR')
     emit_op_int_xor = gen_emit_op_ri('int_xor', 'EOR')
@@ -466,7 +467,11 @@
             assert saveerrloc.is_imm()
             cb.emit_call_release_gil(saveerrloc.value)
         else:
-            cb.emit()
+            effectinfo = descr.get_extra_info()
+            if effectinfo is None or effectinfo.check_can_collect():
+                cb.emit()
+            else:
+                cb.emit_no_collect()
         return fcond
 
     def _genop_same_as(self, op, arglocs, regalloc, fcond):
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -7,7 +7,7 @@
 from rpython.jit.backend.arm import conditions as c
 from rpython.jit.backend.arm import locations
 from rpython.jit.backend.arm.locations import imm, get_fp_offset
-from rpython.jit.backend.arm.helper.regalloc import (prepare_op_by_helper_call,
+from rpython.jit.backend.arm.helper.regalloc import (
                                                     prepare_unary_cmp,
                                                     prepare_op_ri,
                                                     prepare_int_cmp,
@@ -397,9 +397,9 @@
         else:
             self.rm.force_spill_var(var)
 
-    def before_call(self, force_store=[], save_all_regs=False):
-        self.rm.before_call(force_store, save_all_regs)
-        self.vfprm.before_call(force_store, save_all_regs)
+    def before_call(self, save_all_regs=False):
+        self.rm.before_call(save_all_regs)
+        self.vfprm.before_call(save_all_regs)
 
     def _sync_var(self, v):
         if v.type == FLOAT:
@@ -467,6 +467,8 @@
         self.possibly_free_var(op)
         return [reg1, reg2, res]
 
+    prepare_op_uint_mul_high = prepare_op_int_mul
+
     def prepare_op_int_force_ge_zero(self, op, fcond):
         argloc = self.make_sure_var_in_reg(op.getarg(0))
         resloc = self.force_allocate_reg(op, [op.getarg(0)])
@@ -478,10 +480,6 @@
         resloc = self.force_allocate_reg(op)
         return [argloc, imm(numbytes), resloc]
 
-    prepare_op_int_floordiv = prepare_op_by_helper_call('int_floordiv')
-    prepare_op_int_mod = prepare_op_by_helper_call('int_mod')
-    prepare_op_uint_floordiv = prepare_op_by_helper_call('unit_floordiv')
-
     prepare_op_int_and = prepare_op_ri('int_and')
     prepare_op_int_or = prepare_op_ri('int_or')
     prepare_op_int_xor = prepare_op_ri('int_xor')
@@ -554,8 +552,7 @@
     prepare_op_call_f = _prepare_op_call
     prepare_op_call_n = _prepare_op_call
 
-    def _prepare_call(self, op, force_store=[], save_all_regs=False,
-                      first_arg_index=1):
+    def _prepare_call(self, op, save_all_regs=False, first_arg_index=1):
         args = [None] * (op.numargs() + 3)
         calldescr = op.getdescr()
         assert isinstance(calldescr, CallDescr)
@@ -573,17 +570,27 @@
         args[1] = imm(size)
         args[2] = sign_loc
 
-        args[0] = self._call(op, args, force_store, save_all_regs)
+        effectinfo = calldescr.get_extra_info()
+        if save_all_regs:
+            gc_level = 2
+        elif effectinfo is None or effectinfo.check_can_collect():
+            gc_level = 1
+        else:
+            gc_level = 0
+
+        args[0] = self._call(op, args, gc_level)
         return args
 
-    def _call(self, op, arglocs, force_store=[], save_all_regs=False):
-        # spill variables that need to be saved around calls
-        self.vfprm.before_call(force_store, save_all_regs=save_all_regs)
-        if not save_all_regs:
-            gcrootmap = self.cpu.gc_ll_descr.gcrootmap
-            if gcrootmap and gcrootmap.is_shadow_stack:
-                save_all_regs = 2
-        self.rm.before_call(force_store, save_all_regs=save_all_regs)
+    def _call(self, op, arglocs, gc_level):
+        # spill variables that need to be saved around calls:
+        # gc_level == 0: callee cannot invoke the GC
+        # gc_level == 1: can invoke GC, save all regs that contain pointers
+        # gc_level == 2: can force, save all regs
+        save_all_regs = gc_level == 2
+        self.vfprm.before_call(save_all_regs=save_all_regs)
+        if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap:
+            save_all_regs = 2
+        self.rm.before_call(save_all_regs=save_all_regs)
         resloc = self.after_call(op)
         return resloc
 
@@ -1070,7 +1077,7 @@
     def _prepare_op_call_assembler(self, op, fcond):
         locs = self.locs_for_call_assembler(op)
         tmploc = self.get_scratch_reg(INT, selected_reg=r.r0)
-        resloc = self._call(op, locs + [tmploc], save_all_regs=True)
+        resloc = self._call(op, locs + [tmploc], gc_level=2)
         return locs + [resloc, tmploc]
 
     prepare_op_call_assembler_i = _prepare_op_call_assembler
diff --git a/rpython/jit/backend/arm/support.py b/rpython/jit/backend/arm/support.py
deleted file mode 100644
--- a/rpython/jit/backend/arm/support.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rlib.rarithmetic import r_uint
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-
-eci = ExternalCompilationInfo(post_include_bits=["""
-static int pypy__arm_int_div(int a, int b) {
-    return a/b;
-}
-static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) {
-    return a/b;
-}
-static int pypy__arm_int_mod(int a, int b) {
-    return a % b;
-}
-"""])
-
-
-def arm_int_div_emulator(a, b):
-    return int(a / float(b))
-arm_int_div_sign = lltype.Ptr(
-        lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed))
-arm_int_div = rffi.llexternal(
-    "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed,
-                        _callable=arm_int_div_emulator,
-                        compilation_info=eci,
-                        _nowrapper=True, elidable_function=True)
-
-
-def arm_uint_div_emulator(a, b):
-    return r_uint(a) / r_uint(b)
-arm_uint_div_sign = lltype.Ptr(
-        lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned))
-arm_uint_div = rffi.llexternal(
-    "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned,
-                        _callable=arm_uint_div_emulator,
-                        compilation_info=eci,
-                        _nowrapper=True, elidable_function=True)
-
-
-def arm_int_mod_emulator(a, b):
-    sign = 1
-    if a < 0:
-        a = -1 * a
-        sign = -1
-    if b < 0:
-        b = -1 * b
-    res = a % b
-    return sign * res
-arm_int_mod_sign = arm_int_div_sign
-arm_int_mod = rffi.llexternal(
-    "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed,
-                        _callable=arm_int_mod_emulator,
-                        compilation_info=eci,
-                        _nowrapper=True, elidable_function=True)
diff --git a/rpython/jit/backend/arm/test/test_arch.py b/rpython/jit/backend/arm/test/test_arch.py
deleted file mode 100644
--- a/rpython/jit/backend/arm/test/test_arch.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from rpython.jit.backend.arm import support
-
-def test_mod():
-    assert support.arm_int_mod(10, 2) == 0
-    assert support.arm_int_mod(11, 2) == 1
-    assert support.arm_int_mod(11, 3) == 2
-
-def test_mod2():
-    assert support.arm_int_mod(-10, 2) == 0
-    assert support.arm_int_mod(-11, 2) == -1
-    assert support.arm_int_mod(-11, 3) == -2
-
-def test_mod3():
-    assert support.arm_int_mod(10, -2) == 0
-    assert support.arm_int_mod(11, -2) == 1
-    assert support.arm_int_mod(11, -3) == 2
-
-
-def test_div():
-    assert support.arm_int_div(-7, 2) == -3
-    assert support.arm_int_div(9, 2) == 4
-    assert support.arm_int_div(10, 5) == 2
-
diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py
--- a/rpython/jit/backend/arm/test/test_assembler.py
+++ b/rpython/jit/backend/arm/test/test_assembler.py
@@ -193,32 +193,6 @@
         self.a.gen_func_epilog()
         assert run_asm(self.a) == 61
 
-    def test_DIV(self):
-        self.a.gen_func_prolog()
-        self.a.mc.MOV_ri(r.r0.value, 123)
-        self.a.mc.MOV_ri(r.r1.value, 2)
-        self.a.mc.DIV()
-        self.a.gen_func_epilog()
-        assert run_asm(self.a) == 61
-
-    def test_DIV2(self):
-        self.a.gen_func_prolog()
-        self.a.mc.gen_load_int(r.r0.value, -110)
-        self.a.mc.gen_load_int(r.r1.value, 3)
-        self.a.mc.DIV()
-        self.a.gen_func_epilog()
-        assert run_asm(self.a) == -36
-
-    def test_DIV3(self):
-        self.a.gen_func_prolog()
-        self.a.mc.gen_load_int(r.r8.value, 110)
-        self.a.mc.gen_load_int(r.r9.value, -3)
-        self.a.mc.MOV_rr(r.r0.value, r.r8.value)
-        self.a.mc.MOV_rr(r.r1.value, r.r9.value)
-        self.a.mc.DIV()
-        self.a.gen_func_epilog()
-        assert run_asm(self.a) == -36
-
     def test_bl_with_conditional_exec(self):
         functype = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed))
         call_addr = rffi.cast(lltype.Signed, llhelper(functype, callme))
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -574,27 +574,113 @@
             self.assembler.regalloc_mov(reg, to)
         # otherwise it's clean
 
+    def _bc_spill(self, v, new_free_regs):
+        self._sync_var(v)
+        new_free_regs.append(self.reg_bindings.pop(v))
+
     def before_call(self, force_store=[], save_all_regs=0):
-        """ Spill registers before a call, as described by
-        'self.save_around_call_regs'.  Registers are not spilled if
-        they don't survive past the current operation, unless they
-        are listed in 'force_store'.  'save_all_regs' can be 0 (default),
-        1 (save all), or 2 (save default+PTRs).
+        """Spill or move some registers before a call.  By default,
+        this means: for every register in 'self.save_around_call_regs',
+        if there is a variable there and it survives longer than
+        the current operation, then it is spilled/moved somewhere else.
+
+        'save_all_regs' can be 0 (default set of registers), 1 (do that
+        for all registers), or 2 (default + gc ptrs).
+
+        Overview of what we do (the implementation does it differently,
+        for the same result):
+
+        * we first check the set of registers that are free: call it F.
+
+        * possibly_free_vars() is implied for all variables (except
+          the ones listed in force_store): if they don't survive past
+          the current operation, they are forgotten now.  (Their
+          register remain not in F, because they are typically
+          arguments to the call, so they should not be overwritten by
+          the next step.)
+
+        * then for every variable that needs to be spilled/moved: if
+          there is an entry in F that is acceptable, pick it and emit a
+          move.  Otherwise, emit a spill.  Start doing this with the
+          variables that survive the shortest time, to give them a
+          better change to remain in a register---similar algo as
+          _pick_variable_to_spill().
+
+        Note: when a register is moved, it often (but not always) means
+        we could have been more clever and picked a better register in
+        the first place, when we did so earlier.  It is done this way
+        anyway, as a local hack in this function, because on x86 CPUs
+        such register-register moves are almost free.
         """
+        new_free_regs = []
+        move_or_spill = []
+
         for v, reg in self.reg_bindings.items():
-            if v not in force_store and self.longevity[v][1] <= self.position:
+            max_age = self.longevity[v][1]
+            if v not in force_store and max_age <= self.position:
                 # variable dies
                 del self.reg_bindings[v]
-                self.free_regs.append(reg)
+                new_free_regs.append(reg)
                 continue
-            if save_all_regs != 1 and reg not in self.save_around_call_regs:
-                if save_all_regs == 0:
-                    continue    # we don't have to
-                if v.type != REF:
-                    continue    # only save GC pointers
-            self._sync_var(v)
-            del self.reg_bindings[v]
-            self.free_regs.append(reg)
+
+            if save_all_regs == 1:
+                # we need to spill all registers in this mode
+                self._bc_spill(v, new_free_regs)
+                #
+            elif save_all_regs == 2 and v.type == REF:
+                # we need to spill all GC ptrs in this mode
+                self._bc_spill(v, new_free_regs)
+                #
+            elif reg not in self.save_around_call_regs:
+                continue  # in a register like ebx/rbx: it is fine where it is
+                #
+            else:
+                # this is a register like eax/rax, which needs either
+                # spilling or moving.
+                move_or_spill.append((v, max_age))
+
+        if len(move_or_spill) > 0:
+            while len(self.free_regs) > 0:
+                new_reg = self.free_regs.pop()
+                if new_reg in self.save_around_call_regs:
+                    new_free_regs.append(new_reg)    # not this register...
+                    continue
+                # This 'new_reg' is suitable for moving a candidate to.
+                # Pick the one with the smallest max_age.  (This
+                # is one step of a naive sorting algo, slow in theory,
+                # but the list should always be very small so it
+                # doesn't matter.)
+                best_i = 0
+                smallest_max_age = move_or_spill[0][1]
+                for i in range(1, len(move_or_spill)):
+                    max_age = move_or_spill[i][1]
+                    if max_age < smallest_max_age:
+                        best_i = i
+                        smallest_max_age = max_age
+                v, max_age = move_or_spill.pop(best_i)
+                # move from 'reg' to 'new_reg'
+                reg = self.reg_bindings[v]
+                if not we_are_translated():
+                    if move_or_spill:
+                        assert max_age <= min([_a for _, _a in move_or_spill])
+                    assert reg in self.save_around_call_regs
+                    assert new_reg not in self.save_around_call_regs
+                self.assembler.regalloc_mov(reg, new_reg)
+                self.reg_bindings[v] = new_reg    # change the binding
+                new_free_regs.append(reg)
+                #
+                if len(move_or_spill) == 0:
+                    break
+            else:
+                # no more free registers to move to, spill the rest
+                for v, max_age in move_or_spill:
+                    self._bc_spill(v, new_free_regs)
+
+        # re-add registers in 'new_free_regs', but in reverse order,
+        # so that the last ones (added just above, from
+        # save_around_call_regs) are picked last by future '.pop()'
+        while len(new_free_regs) > 0:
+            self.free_regs.append(new_free_regs.pop())
 
     def after_call(self, v):
         """ Adjust registers according to the result of the call,
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
@@ -496,22 +496,6 @@
         self.interpret(ops, [s, ord('a')])
         assert s[1] == 'a'
 
-    def test_division_optimized(self):
-        ops = '''
-        [i7, i6]
-        label(i7, i6, descr=targettoken)
-        i18 = int_floordiv(i7, i6)
-        i19 = int_xor(i7, i6)
-        i21 = int_lt(i19, 0)
-        i22 = int_mod(i7, i6)
-        i23 = int_is_true(i22)
-        i24 = int_eq(i6, 4)
-        guard_false(i24) [i18]
-        jump(i18, i6, descr=targettoken)
-        '''
-        self.interpret(ops, [10, 4])
-        assert self.getint(0) == 2
-        # FIXME: Verify that i19 - i23 are removed
 
 class TestRegallocFloats(BaseTestRegalloc):
     def setup_class(cls):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -62,6 +62,12 @@
         else:
             self.mc.mulld(res.value, l0.value, l1.value)
 
+    def emit_uint_mul_high(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        assert not l0.is_imm()
+        assert not l1.is_imm()
+        self.mc.mulhdu(res.value, l0.value, l1.value)
+
     def do_emit_int_binary_ovf(self, op, arglocs):
         l0, l1, res = arglocs[0], arglocs[1], arglocs[2]
         self.mc.load_imm(r.SCRATCH, 0)
@@ -80,24 +86,6 @@
         else:
             self.mc.mulldox(*self.do_emit_int_binary_ovf(op, arglocs))
 
-    def emit_int_floordiv(self, op, arglocs, regalloc):
-        l0, l1, res = arglocs
-        if IS_PPC_32:
-            self.mc.divw(res.value, l0.value, l1.value)
-        else:
-            self.mc.divd(res.value, l0.value, l1.value)
-
-    def emit_int_mod(self, op, arglocs, regalloc):
-        l0, l1, res = arglocs
-        if IS_PPC_32:
-            self.mc.divw(r.r0.value, l0.value, l1.value)
-            self.mc.mullw(r.r0.value, r.r0.value, l1.value)
-        else:
-            self.mc.divd(r.r0.value, l0.value, l1.value)
-            self.mc.mulld(r.r0.value, r.r0.value, l1.value)
-        self.mc.subf(r.r0.value, r.r0.value, l0.value)
-        self.mc.mr(res.value, r.r0.value)
-
     def emit_int_and(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.and_(res.value, l0.value, l1.value)
@@ -130,13 +118,6 @@
             self.mc.srw(res.value, l0.value, l1.value)
         else:
             self.mc.srd(res.value, l0.value, l1.value)
-    
-    def emit_uint_floordiv(self, op, arglocs, regalloc):
-        l0, l1, res = arglocs
-        if IS_PPC_32:
-            self.mc.divwu(res.value, l0.value, l1.value)
-        else:
-            self.mc.divdu(res.value, l0.value, l1.value)
 
     emit_int_le = gen_emit_cmp_op(c.LE)
     emit_int_lt = gen_emit_cmp_op(c.LT)
@@ -622,7 +603,11 @@
             assert saveerrloc.is_imm()
             cb.emit_call_release_gil(saveerrloc.value)
         else:
-            cb.emit()
+            effectinfo = descr.get_extra_info()
+            if effectinfo is None or effectinfo.check_can_collect():
+                cb.emit()
+            else:
+                cb.emit_no_collect()
 
     def _genop_call(self, op, arglocs, regalloc):
         oopspecindex = regalloc.get_oopspecindex(op)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager,
                                                     TempVar, compute_vars_longevity,
                                                     BaseRegalloc)
+from rpython.jit.backend.llsupport.descr import CallDescr
 from rpython.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32)
 from rpython.jit.codewriter import longlong
 from rpython.jit.backend.ppc.jump import (remap_frame_layout,
@@ -369,9 +370,9 @@
         # This operation is used only for testing
         self.force_spill_var(op.getarg(0))
 
-    def before_call(self, force_store=[], save_all_regs=False):
-        self.rm.before_call(force_store, save_all_regs)
-        self.fprm.before_call(force_store, save_all_regs)
+    def before_call(self, save_all_regs=False):
+        self.rm.before_call(save_all_regs)
+        self.fprm.before_call(save_all_regs)
 
     def after_call(self, v):
         if v.type == FLOAT:
@@ -432,15 +433,13 @@
     prepare_int_mul = helper.prepare_int_add_or_mul
     prepare_nursery_ptr_increment = prepare_int_add
 
-    prepare_int_floordiv = helper.prepare_binary_op
-    prepare_int_mod = helper.prepare_binary_op
     prepare_int_and = helper.prepare_binary_op
     prepare_int_or = helper.prepare_binary_op
     prepare_int_xor = helper.prepare_binary_op
     prepare_int_lshift = helper.prepare_binary_op
     prepare_int_rshift = helper.prepare_binary_op
     prepare_uint_rshift = helper.prepare_binary_op
-    prepare_uint_floordiv = helper.prepare_binary_op
+    prepare_uint_mul_high = helper.prepare_binary_op
 
     prepare_int_add_ovf = helper.prepare_binary_op
     prepare_int_sub_ovf = helper.prepare_binary_op
@@ -758,7 +757,7 @@
         src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2))
         dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3))
         length_loc  = self.ensure_reg_or_any_imm(op.getarg(4))
-        self._spill_before_call(save_all_regs=False)
+        self._spill_before_call(gc_level=0)
         return [src_ptr_loc, dst_ptr_loc,
                 src_ofs_loc, dst_ofs_loc, length_loc]
 
@@ -791,13 +790,15 @@
     prepare_call_f = _prepare_call
     prepare_call_n = _prepare_call
 
-    def _spill_before_call(self, save_all_regs=False):
-        # spill variables that need to be saved around calls
+    def _spill_before_call(self, gc_level):
+        # spill variables that need to be saved around calls:
+        # gc_level == 0: callee cannot invoke the GC
+        # gc_level == 1: can invoke GC, save all regs that contain pointers
+        # gc_level == 2: can force, save all regs
+        save_all_regs = gc_level == 2
         self.fprm.before_call(save_all_regs=save_all_regs)
-        if not save_all_regs:
-            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
-            if gcrootmap and gcrootmap.is_shadow_stack:
-                save_all_regs = 2
+        if gc_level == 1 and self.cpu.gc_ll_descr.gcrootmap:
+            save_all_regs = 2
         self.rm.before_call(save_all_regs=save_all_regs)
 
     def _prepare_call(self, op, save_all_regs=False):
@@ -805,7 +806,18 @@
         args.append(None)
         for i in range(op.numargs()):
             args.append(self.loc(op.getarg(i)))
-        self._spill_before_call(save_all_regs)
+
+        calldescr = op.getdescr()
+        assert isinstance(calldescr, CallDescr)
+        effectinfo = calldescr.get_extra_info()
+        if save_all_regs:
+            gc_level = 2
+        elif effectinfo is None or effectinfo.check_can_collect():
+            gc_level = 1
+        else:
+            gc_level = 0
+        self._spill_before_call(gc_level=gc_level)
+
         if op.type != VOID:
             resloc = self.after_call(op)
             args[0] = resloc
@@ -934,7 +946,7 @@
 
     def _prepare_call_assembler(self, op):
         locs = self.locs_for_call_assembler(op)
-        self._spill_before_call(save_all_regs=True)
+        self._spill_before_call(gc_level=2)
         if op.type != VOID:
             resloc = self.after_call(op)
         else:
diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -532,6 +532,7 @@
             rop.INT_AND,
             rop.INT_OR,
             rop.INT_XOR,
+            rop.UINT_MUL_HIGH,
             ]:
     OPERATIONS.append(BinaryOperation(_op))
 
@@ -548,8 +549,8 @@
             ]:
     OPERATIONS.append(BinaryOperation(_op, boolres=True))
 
-OPERATIONS.append(BinaryOperation(rop.INT_FLOORDIV, ~3, 2))
-OPERATIONS.append(BinaryOperation(rop.INT_MOD, ~3, 2))
+#OPERATIONS.append(BinaryOperation(rop.INT_FLOORDIV, ~3, 2))
+#OPERATIONS.append(BinaryOperation(rop.INT_MOD, ~3, 2))
 OPERATIONS.append(BinaryOperation(rop.INT_RSHIFT, LONG_BIT-1))
 OPERATIONS.append(BinaryOperation(rop.INT_LSHIFT, LONG_BIT-1))
 OPERATIONS.append(BinaryOperation(rop.UINT_RSHIFT, LONG_BIT-1))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1289,6 +1289,9 @@
     genop_float_mul = _binaryop('MULSD')
     genop_float_truediv = _binaryop('DIVSD')
 
+    def genop_uint_mul_high(self, op, arglocs, result_loc):
+        self.mc.MUL(arglocs[0])
+
     def genop_int_and(self, op, arglocs, result_loc):
         arg1 = arglocs[1]
         if IS_X86_64 and (isinstance(arg1, ImmedLoc) and
@@ -1444,20 +1447,6 @@
         self.mov(imm0, resloc)
         self.mc.CMOVNS(resloc, arglocs[0])
 
-    def genop_int_mod(self, op, arglocs, resloc):
-        if IS_X86_32:
-            self.mc.CDQ()
-        elif IS_X86_64:
-            self.mc.CQO()
-
-        self.mc.IDIV_r(ecx.value)
-
-    genop_int_floordiv = genop_int_mod
-
-    def genop_uint_floordiv(self, op, arglocs, resloc):
-        self.mc.XOR_rr(edx.value, edx.value)
-        self.mc.DIV_r(ecx.value)
-
     genop_llong_add = _binaryop("PADDQ")
     genop_llong_sub = _binaryop("PSUBQ")
     genop_llong_and = _binaryop("PAND")
@@ -2123,7 +2112,11 @@
             assert isinstance(saveerrloc, ImmedLoc)
             cb.emit_call_release_gil(saveerrloc.value)
         else:
-            cb.emit()
+            effectinfo = descr.get_extra_info()
+            if effectinfo is None or effectinfo.check_can_collect():
+                cb.emit()
+            else:
+                cb.emit_no_collect()
 
     def _store_force_index(self, guard_op):
         assert (guard_op.getopnum() == rop.GUARD_NOT_FORCED or
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -561,6 +561,27 @@
     consider_int_sub_ovf = _consider_binop
     consider_int_add_ovf = _consider_binop_symm
 
+    def consider_uint_mul_high(self, op):
+        arg1, arg2 = op.getarglist()
+        # should support all cases, but is optimized for (box, const)
+        if isinstance(arg1, Const):
+            arg1, arg2 = arg2, arg1
+        self.rm.make_sure_var_in_reg(arg2, selected_reg=eax)
+        l1 = self.loc(arg1)
+        # l1 is a register != eax, or stack_bp; or, just possibly, it
+        # can be == eax if arg1 is arg2
+        assert not isinstance(l1, ImmedLoc)
+        assert l1 is not eax or arg1 is arg2
+        #
+        # eax will be trash after the operation
+        self.rm.possibly_free_var(arg2)
+        tmpvar = TempVar()
+        self.rm.force_allocate_reg(tmpvar, selected_reg=eax)
+        self.rm.possibly_free_var(tmpvar)
+        #
+        self.rm.force_allocate_reg(op, selected_reg=edx)
+        self.perform(op, [l1], edx)
+
     def consider_int_neg(self, op):
         res = self.rm.force_result_in_reg(op, op.getarg(0))
         self.perform(op, [res], res)
@@ -585,29 +606,6 @@
     consider_int_rshift  = consider_int_lshift
     consider_uint_rshift = consider_int_lshift
 
-    def _consider_int_div_or_mod(self, op, resultreg, trashreg):
-        l0 = self.rm.make_sure_var_in_reg(op.getarg(0), selected_reg=eax)
-        l1 = self.rm.make_sure_var_in_reg(op.getarg(1), selected_reg=ecx)
-        l2 = self.rm.force_allocate_reg(op, selected_reg=resultreg)
-        # the register (eax or edx) not holding what we are looking for
-        # will be just trash after that operation
-        tmpvar = TempVar()
-        self.rm.force_allocate_reg(tmpvar, selected_reg=trashreg)
-        assert l0 is eax
-        assert l1 is ecx
-        assert l2 is resultreg
-        self.rm.possibly_free_var(tmpvar)
-
-    def consider_int_mod(self, op):
-        self._consider_int_div_or_mod(op, edx, eax)
-        self.perform(op, [eax, ecx], edx)
-
-    def consider_int_floordiv(self, op):
-        self._consider_int_div_or_mod(op, eax, edx)
-        self.perform(op, [eax, ecx], eax)
-
-    consider_uint_floordiv = consider_int_floordiv
-
     def _consider_compop(self, op):
         vx = op.getarg(0)
         vy = op.getarg(1)
@@ -797,22 +795,22 @@
         else:
             self._consider_call(op)
 
-    def _call(self, op, arglocs, force_store=[], guard_not_forced=False):
+    def _call(self, op, arglocs, gc_level):
         # we need to save registers on the stack:
         #
         #  - at least the non-callee-saved registers
         #
-        #  - we assume that any call can collect, and we
-        #    save also the callee-saved registers that contain GC pointers
+        #  - if gc_level > 0, we save also the callee-saved registers that
+        #    contain GC pointers
         #
-        #  - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs
-        #    anyway, in case we need to do cpu.force().  The issue is that
-        #    grab_frame_values() would not be able to locate values in
-        #    callee-saved registers.
+        #  - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER.  We
+        #    have to save all regs anyway, in case we need to do
+        #    cpu.force().  The issue is that grab_frame_values() would
+        #    not be able to locate values in callee-saved registers.
         #
-        save_all_regs = guard_not_forced
-        self.xrm.before_call(force_store, save_all_regs=save_all_regs)
-        if not save_all_regs:
+        save_all_regs = gc_level == 2
+        self.xrm.before_call(save_all_regs=save_all_regs)
+        if gc_level == 1:
             gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             # we save all the registers for shadowstack and asmgcc for now
             # --- for asmgcc too: we can't say "register x is a gc ref"
@@ -820,7 +818,7 @@
             # more for now.
             if gcrootmap: # and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
-        self.rm.before_call(force_store, save_all_regs=save_all_regs)
+        self.rm.before_call(save_all_regs=save_all_regs)
         if op.type != 'v':
             if op.type == FLOAT:
                 resloc = self.xrm.after_call(op)
@@ -840,9 +838,18 @@
             sign_loc = imm1
         else:
             sign_loc = imm0
+        #
+        effectinfo = calldescr.get_extra_info()
+        if guard_not_forced:
+            gc_level = 2
+        elif effectinfo is None or effectinfo.check_can_collect():
+            gc_level = 1
+        else:
+            gc_level = 0
+        #
         self._call(op, [imm(size), sign_loc] +
                        [self.loc(op.getarg(i)) for i in range(op.numargs())],
-                   guard_not_forced=guard_not_forced)
+                   gc_level=gc_level)
 
     def _consider_real_call(self, op):
         effectinfo = op.getdescr().get_extra_info()
@@ -901,7 +908,7 @@
 
     def _consider_call_assembler(self, op):
         locs = self.locs_for_call_assembler(op)
-        self._call(op, locs, guard_not_forced=True)
+        self._call(op, locs, gc_level=2)
     consider_call_assembler_i = _consider_call_assembler
     consider_call_assembler_r = _consider_call_assembler
     consider_call_assembler_f = _consider_call_assembler
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -641,6 +641,7 @@
     SUB = _binaryop('SUB')
     IMUL = _binaryop('IMUL')
     NEG = _unaryop('NEG')
+    MUL = _unaryop('MUL')
 
     CMP = _binaryop('CMP')
     CMP16 = _binaryop('CMP16')
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -558,6 +558,9 @@
     DIV_r = insn(rex_w, '\xF7', register(1), '\xF0')
     IDIV_r = insn(rex_w, '\xF7', register(1), '\xF8')
 
+    MUL_r = insn(rex_w, '\xF7', orbyte(4<<3), register(1), '\xC0')
+    MUL_b = insn(rex_w, '\xF7', orbyte(4<<3), stack_bp(1))
+
     IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0')
     IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2))
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -14,6 +14,7 @@
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
 from rpython.translator.backendopt.graphanalyze import DependencyTracker
+from rpython.translator.backendopt.collectanalyze import CollectAnalyzer
 
 
 class CallControl(object):
@@ -37,9 +38,9 @@
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
             self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator)
             self.randomeffects_analyzer = RandomEffectsAnalyzer(translator)
-            self.seen = DependencyTracker(self.readwrite_analyzer)
-        else:
-            self.seen = None
+            self.collect_analyzer = CollectAnalyzer(translator)
+            self.seen_rw = DependencyTracker(self.readwrite_analyzer)
+            self.seen_gc = DependencyTracker(self.collect_analyzer)
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
@@ -294,9 +295,9 @@
                     "but the function has no result" % (op, ))
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
-            extradescr,

From pypy.commits at gmail.com  Mon Jun 20 18:16:44 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Let's try this.
Message-ID: <57686b4c.e7c9c20a.9dfba.ffffa6cc@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85273:577eaa27faf2
Date: 2016-05-24 21:16 -0400
http://bitbucket.org/pypy/pypy/changeset/577eaa27faf2/

Log:	Let's try this.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -111,6 +111,13 @@
             self.n_overflow = 0
             self.last_ticks = 0
     time_state = TimeState()
+    from rpython.rlib.rdynload import GetModuleHandle, dlsym
+    hKernel32 = GetModuleHandle("KERNEL32")
+    try:
+        dlsym(hKernel32, 'GetFinalPathNameByHandleW')
+        HAS_GETTICKCOUNT64 = True
+    except KeyError:
+        HAS_GETTICKCOUNT64 = False
 
 _includes = ["time.h"]
 if _POSIX:
@@ -194,7 +201,6 @@
 
 CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC
 HAS_CLOCK_GETTIME = cConfig.has_clock_gettime
-HAS_GETTICKCOUNT64 = cConfig.has_gettickcount64
 clock_t = cConfig.clock_t
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)

From pypy.commits at gmail.com  Mon Jun 20 18:16:56 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merge with upstream.
Message-ID: <57686b58.6372c20a.3456f.4481@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85279:ae756a654002
Date: 2016-05-29 11:56 -0400
http://bitbucket.org/pypy/pypy/changeset/ae756a654002/

Log:	Merge with upstream.

diff too long, truncating to 2000 out of 3066 lines

diff --git a/lib-python/3/ensurepip/__init__.py b/lib-python/3/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/ensurepip/__init__.py
@@ -0,0 +1,210 @@
+import os
+import os.path
+import pkgutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "21.2.1"
+
+_PIP_VERSION = "8.1.2"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(*, root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=False,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+
+def _uninstall_helper(*, verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y", "--disable-pip-version-check"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=False,
+        help=("Make a default pip install, installing the unqualified pip "
+              "and easy_install in addition to the versioned scripts"),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/3/ensurepip/__main__.py b/lib-python/3/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/3/ensurepip/_bundled/pip-8.1.2-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-8.1.2-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cc49227a0c7e13757f4863a9b7ace1eb56c3ce61
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_bundled/setuptools-21.2.1-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-21.2.1-py2.py3-none-any.whl
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..fe36464f79ba87960c33f3bdff817deb9e4e5f7c
GIT binary patch

[cut]

diff --git a/lib-python/3/ensurepip/_uninstall.py b/lib-python/3/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/3/importlib/_bootstrap.py b/lib-python/3/importlib/_bootstrap.py
--- a/lib-python/3/importlib/_bootstrap.py
+++ b/lib-python/3/importlib/_bootstrap.py
@@ -768,7 +768,7 @@
         else:
             registry_key = cls.REGISTRY_KEY
         key = registry_key.format(fullname=fullname,
-                                  sys_version=sys.version[:3])
+                                  sys_version='%d.%d' % sys.version_info[:2])
         try:
             with cls._open_registry(key) as hkey:
                 filepath = _winreg.QueryValue(hkey, "")
diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py
--- a/lib-python/3/subprocess.py
+++ b/lib-python/3/subprocess.py
@@ -976,15 +976,18 @@
             c2pread, c2pwrite = -1, -1
             errread, errwrite = -1, -1
 
+            ispread = False
             if stdin is None:
                 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _winapi.CreatePipe(None, 0)
                     p2cread = Handle(p2cread)
                     _winapi.CloseHandle(_)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
                 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
+                ispread = True
             elif stdin == DEVNULL:
                 p2cread = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdin, int):
@@ -992,17 +995,20 @@
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _winapi.CreatePipe(None, 0)
                     c2pwrite = Handle(c2pwrite)
                     _winapi.CloseHandle(_)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
                 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
+                ispwrite = True
             elif stdout == DEVNULL:
                 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdout, int):
@@ -1010,17 +1016,20 @@
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _winapi.CreatePipe(None, 0)
                     errwrite = Handle(errwrite)
                     _winapi.CloseHandle(_)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _winapi.CreatePipe(None, 0)
                 errread, errwrite = Handle(errread), Handle(errwrite)
+                ispwrite = True
             elif stderr == STDOUT:
                 errwrite = c2pwrite
             elif stderr == DEVNULL:
@@ -1030,19 +1039,23 @@
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
 
             return (p2cread, p2cwrite,
                     c2pread, c2pwrite,
                     errread, errwrite)
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             h = _winapi.DuplicateHandle(
                 _winapi.GetCurrentProcess(), handle,
                 _winapi.GetCurrentProcess(), 0, 1,
                 _winapi.DUPLICATE_SAME_ACCESS)
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
+                handle.Close()
             return Handle(h)
 
 
diff --git a/lib-python/3/test/test_ensurepip.py b/lib-python/3/test/test_ensurepip.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_ensurepip.py
@@ -0,0 +1,360 @@
+import unittest
+import unittest.mock
+import test.support
+import os
+import os.path
+import contextlib
+import sys
+
+import ensurepip
+import ensurepip._uninstall
+
+# pip currently requires ssl support, so we ensure we handle
+# it being missing (http://bugs.python.org/issue19744)
+ensurepip_no_ssl = test.support.import_fresh_module("ensurepip",
+                                                    blocked=["ssl"])
+try:
+    import ssl
+except ImportError:
+    def requires_usable_pip(f):
+        deco = unittest.skip(ensurepip._MISSING_SSL_MESSAGE)
+        return deco(f)
+else:
+    def requires_usable_pip(f):
+        return f
+
+class TestEnsurePipVersion(unittest.TestCase):
+
+    def test_returns_version(self):
+        self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
+
+class EnsurepipMixin:
+
+    def setUp(self):
+        run_pip_patch = unittest.mock.patch("ensurepip._run_pip")
+        self.run_pip = run_pip_patch.start()
+        self.addCleanup(run_pip_patch.stop)
+
+        # Avoid side effects on the actual os module
+        real_devnull = os.devnull
+        os_patch = unittest.mock.patch("ensurepip.os")
+        patched_os = os_patch.start()
+        self.addCleanup(os_patch.stop)
+        patched_os.devnull = real_devnull
+        patched_os.path = os.path
+        self.os_environ = patched_os.environ = os.environ.copy()
+
+
+class TestBootstrap(EnsurepipMixin, unittest.TestCase):
+
+    @requires_usable_pip
+    def test_basic_bootstrapping(self):
+        ensurepip.bootstrap()
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+        additional_paths = self.run_pip.call_args[0][1]
+        self.assertEqual(len(additional_paths), 2)
+
+    @requires_usable_pip
+    def test_bootstrapping_with_root(self):
+        ensurepip.bootstrap(root="/foo/bar/")
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--root", "/foo/bar/",
+                "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_user(self):
+        ensurepip.bootstrap(user=True)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--user", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_upgrade(self):
+        ensurepip.bootstrap(upgrade=True)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--upgrade", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_verbosity_1(self):
+        ensurepip.bootstrap(verbosity=1)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "-v", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_verbosity_2(self):
+        ensurepip.bootstrap(verbosity=2)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "-vv", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_verbosity_3(self):
+        ensurepip.bootstrap(verbosity=3)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "-vvv", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    @requires_usable_pip
+    def test_bootstrapping_with_regular_install(self):
+        ensurepip.bootstrap()
+        self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "install")
+
+    @requires_usable_pip
+    def test_bootstrapping_with_alt_install(self):
+        ensurepip.bootstrap(altinstall=True)
+        self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "altinstall")
+
+    @requires_usable_pip
+    def test_bootstrapping_with_default_pip(self):
+        ensurepip.bootstrap(default_pip=True)
+        self.assertNotIn("ENSUREPIP_OPTIONS", self.os_environ)
+
+    def test_altinstall_default_pip_conflict(self):
+        with self.assertRaises(ValueError):
+            ensurepip.bootstrap(altinstall=True, default_pip=True)
+        self.assertFalse(self.run_pip.called)
+
+    @requires_usable_pip
+    def test_pip_environment_variables_removed(self):
+        # ensurepip deliberately ignores all pip environment variables
+        # See http://bugs.python.org/issue19734 for details
+        self.os_environ["PIP_THIS_SHOULD_GO_AWAY"] = "test fodder"
+        ensurepip.bootstrap()
+        self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ)
+
+    @requires_usable_pip
+    def test_pip_config_file_disabled(self):
+        # ensurepip deliberately ignores the pip config file
+        # See http://bugs.python.org/issue20053 for details
+        ensurepip.bootstrap()
+        self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
+
+ at contextlib.contextmanager
+def fake_pip(version=ensurepip._PIP_VERSION):
+    if version is None:
+        pip = None
+    else:
+        class FakePip():
+            __version__ = version
+        pip = FakePip()
+    sentinel = object()
+    orig_pip = sys.modules.get("pip", sentinel)
+    sys.modules["pip"] = pip
+    try:
+        yield pip
+    finally:
+        if orig_pip is sentinel:
+            del sys.modules["pip"]
+        else:
+            sys.modules["pip"] = orig_pip
+
+class TestUninstall(EnsurepipMixin, unittest.TestCase):
+
+    def test_uninstall_skipped_when_not_installed(self):
+        with fake_pip(None):
+            ensurepip._uninstall_helper()
+        self.assertFalse(self.run_pip.called)
+
+    def test_uninstall_skipped_with_warning_for_wrong_version(self):
+        with fake_pip("not a valid version"):
+            with test.support.captured_stderr() as stderr:
+                ensurepip._uninstall_helper()
+        warning = stderr.getvalue().strip()
+        self.assertIn("only uninstall a matching version", warning)
+        self.assertFalse(self.run_pip.called)
+
+
+    @requires_usable_pip
+    def test_uninstall(self):
+        with fake_pip():
+            ensurepip._uninstall_helper()
+
+        self.run_pip.assert_called_once_with(
+            [
+                "uninstall", "-y", "--disable-pip-version-check", "pip",
+                "setuptools",
+            ]
+        )
+
+    @requires_usable_pip
+    def test_uninstall_with_verbosity_1(self):
+        with fake_pip():
+            ensurepip._uninstall_helper(verbosity=1)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "uninstall", "-y", "--disable-pip-version-check", "-v", "pip",
+                "setuptools",
+            ]
+        )
+
+    @requires_usable_pip
+    def test_uninstall_with_verbosity_2(self):
+        with fake_pip():
+            ensurepip._uninstall_helper(verbosity=2)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "uninstall", "-y", "--disable-pip-version-check", "-vv", "pip",
+                "setuptools",
+            ]
+        )
+
+    @requires_usable_pip
+    def test_uninstall_with_verbosity_3(self):
+        with fake_pip():
+            ensurepip._uninstall_helper(verbosity=3)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "uninstall", "-y", "--disable-pip-version-check", "-vvv",
+                "pip", "setuptools",
+            ]
+        )
+
+    @requires_usable_pip
+    def test_pip_environment_variables_removed(self):
+        # ensurepip deliberately ignores all pip environment variables
+        # See http://bugs.python.org/issue19734 for details
+        self.os_environ["PIP_THIS_SHOULD_GO_AWAY"] = "test fodder"
+        with fake_pip():
+            ensurepip._uninstall_helper()
+        self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ)
+
+    @requires_usable_pip
+    def test_pip_config_file_disabled(self):
+        # ensurepip deliberately ignores the pip config file
+        # See http://bugs.python.org/issue20053 for details
+        with fake_pip():
+            ensurepip._uninstall_helper()
+        self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
+
+
+class TestMissingSSL(EnsurepipMixin, unittest.TestCase):
+
+    def setUp(self):
+        sys.modules["ensurepip"] = ensurepip_no_ssl
+        @self.addCleanup
+        def restore_module():
+            sys.modules["ensurepip"] = ensurepip
+        super().setUp()
+
+    def test_bootstrap_requires_ssl(self):
+        self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder"
+        with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"):
+            ensurepip_no_ssl.bootstrap()
+        self.assertFalse(self.run_pip.called)
+        self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ)
+
+    def test_uninstall_requires_ssl(self):
+        self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder"
+        with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"):
+            with fake_pip():
+                ensurepip_no_ssl._uninstall_helper()
+        self.assertFalse(self.run_pip.called)
+        self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ)
+
+    def test_main_exits_early_with_warning(self):
+        with test.support.captured_stderr() as stderr:
+            ensurepip_no_ssl._main(["--version"])
+        warning = stderr.getvalue().strip()
+        self.assertTrue(warning.endswith("requires SSL/TLS"), warning)
+        self.assertFalse(self.run_pip.called)
+
+# Basic testing of the main functions and their argument parsing
+
+EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION
+
+class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase):
+
+    @requires_usable_pip
+    def test_bootstrap_version(self):
+        with test.support.captured_stderr() as stdout:
+            with self.assertRaises(SystemExit):
+                ensurepip._main(["--version"])
+        result = stdout.getvalue().strip()
+        self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+        self.assertFalse(self.run_pip.called)
+
+    @requires_usable_pip
+    def test_basic_bootstrapping(self):
+        ensurepip._main([])
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+        additional_paths = self.run_pip.call_args[0][1]
+        self.assertEqual(len(additional_paths), 2)
+
+class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase):
+
+    def test_uninstall_version(self):
+        with test.support.captured_stderr() as stdout:
+            with self.assertRaises(SystemExit):
+                ensurepip._uninstall._main(["--version"])
+        result = stdout.getvalue().strip()
+        self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+        self.assertFalse(self.run_pip.called)
+
+    @requires_usable_pip
+    def test_basic_uninstall(self):
+        with fake_pip():
+            ensurepip._uninstall._main([])
+
+        self.run_pip.assert_called_once_with(
+            [
+                "uninstall", "-y", "--disable-pip-version-check", "pip",
+                "setuptools",
+            ]
+        )
+
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -196,6 +196,7 @@
     RegrTest('test_dummy_threading.py', core=True),
     RegrTest('test_dynamic.py'),
     RegrTest('test_email', skip="XXX is a directory"),
+    RegrTest('test_ensurepip.py'),
     RegrTest('test_enumerate.py', core=True),
     RegrTest('test_eof.py', core=True),
     RegrTest('test_epoll.py'),
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -161,6 +161,15 @@
 _codecs.register_error('_decimal_encode', _handle_decimaldigits)
 
 
+def _unsafe_check(name, lo, hi, value):
+    if not -_sys.maxsize-1 <= value <= _sys.maxsize:
+        raise OverflowError(
+            "Python int too large to convert to C ssize_t")
+    if not lo <= value <= hi:
+        raise ValueError("valid range for unsafe %s is [%d, %d]" %
+                         (name, lo, hi))
+
+
 # Decimal class
 
 _DEC_MINALLOC = 4
@@ -298,7 +307,8 @@
                 raise ValueError("exponent must be an integer")
             if not -_sys.maxsize-1 <= exponent <= _sys.maxsize:
                 # Compatibility with CPython
-                raise OverflowError()
+                raise OverflowError(
+                    "Python int too large to convert to C ssize_t")
 
         # coefficients
         if not digits and not is_special:
@@ -1501,6 +1511,19 @@
             _mpdec.mpd_free(output)
         return result.decode()
 
+    if _sys.maxsize < 2**63-1:
+        def _unsafe_setprec(self, value):
+            _unsafe_check('prec', 1, 1070000000, value)
+            self._ctx.prec = value
+
+        def _unsafe_setemin(self, value):
+            _unsafe_check('emin', -1070000000, 0, value)
+            self._ctx.emin = value
+
+        def _unsafe_setemax(self, value):
+            _unsafe_check('emax', 0, 1070000000, value)
+            self._ctx.emax = value
+
 
 class _SignalDict(_collections.abc.MutableMapping):
 
diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vccompat.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2008-2016 Stefan Krah. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#ifndef VCCOMPAT_H
+#define VCCOMPAT_H
+
+
+/* Visual C fixes: no stdint.h, no snprintf ... */
+#ifdef _MSC_VER
+  #include "vcstdint.h"
+  #undef inline
+  #define inline __inline
+  #undef random
+  #define random rand
+  #undef srandom
+  #define srandom srand
+  #undef snprintf
+  #define snprintf sprintf_s
+  #define HAVE_SNPRINTF
+  #undef strncasecmp
+  #define strncasecmp _strnicmp
+  #undef strcasecmp
+  #define strcasecmp _stricmp
+  #undef strtoll
+  #define strtoll _strtoi64
+  #define strdup _strdup
+  #define PRIi64 "I64i"
+  #define PRIu64 "I64u"
+  #define PRIi32 "I32i"
+  #define PRIu32 "I32u"
+#endif
+
+
+#endif /* VCCOMPAT_H */
+
+
+
diff --git a/lib_pypy/_libmpdec/vcdiv64.asm b/lib_pypy/_libmpdec/vcdiv64.asm
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vcdiv64.asm
@@ -0,0 +1,48 @@
+;
+; Copyright (c) 2008-2016 Stefan Krah. All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+;
+; 1. Redistributions of source code must retain the above copyright
+;    notice, this list of conditions and the following disclaimer.
+;
+; 2. Redistributions in binary form must reproduce the above copyright
+;    notice, this list of conditions and the following disclaimer in the
+;    documentation and/or other materials provided with the distribution.
+;
+; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+; ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+; SUCH DAMAGE.
+;
+
+
+PUBLIC    _mpd_div_words
+_TEXT    SEGMENT
+q$ = 8
+r$ = 16
+hi$ = 24
+lo$ = 32
+d$ = 40
+_mpd_div_words PROC
+    mov    r10, rdx
+    mov    rdx, r8
+    mov    rax, r9
+    div    QWORD PTR d$[rsp]
+    mov    QWORD PTR [r10], rdx
+    mov    QWORD PTR [rcx], rax
+    ret    0
+_mpd_div_words ENDP
+_TEXT    ENDS
+END
+
+
diff --git a/lib_pypy/_libmpdec/vcstdint.h b/lib_pypy/_libmpdec/vcstdint.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_libmpdec/vcstdint.h
@@ -0,0 +1,232 @@
+// ISO C9x  compliant stdint.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
+//
+//  Copyright (c) 2006-2008 Alexander Chemeris
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   1. Redistributions of source code must retain the above copyright notice,
+//      this list of conditions and the following disclaimer.
+//
+//   2. Redistributions in binary form must reproduce the above copyright
+//      notice, this list of conditions and the following disclaimer in the
+//      documentation and/or other materials provided with the distribution.
+//
+//   3. The name of the author may be used to endorse or promote products
+//      derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_STDINT_H_ // [
+#define _MSC_STDINT_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include 
+
+// For Visual Studio 6 in C++ mode wrap  include with 'extern "C++" {}'
+// or compiler give many errors like this:
+//   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
+#if (_MSC_VER < 1300) && defined(__cplusplus)
+   extern "C++" {
+#endif
+#     include 
+#if (_MSC_VER < 1300) && defined(__cplusplus)
+   }
+#endif
+
+// Define _W64 macros to mark types changing their size, like intptr_t.
+#ifndef _W64
+#  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
+#     define _W64 __w64
+#  else
+#     define _W64
+#  endif
+#endif
+
+
+// 7.18.1 Integer types
+
+// 7.18.1.1 Exact-width integer types
+typedef __int8            int8_t;
+typedef __int16           int16_t;
+typedef __int32           int32_t;
+typedef __int64           int64_t;
+typedef unsigned __int8   uint8_t;
+typedef unsigned __int16  uint16_t;
+typedef unsigned __int32  uint32_t;
+typedef unsigned __int64  uint64_t;
+
+// 7.18.1.2 Minimum-width integer types
+typedef int8_t    int_least8_t;
+typedef int16_t   int_least16_t;
+typedef int32_t   int_least32_t;
+typedef int64_t   int_least64_t;
+typedef uint8_t   uint_least8_t;
+typedef uint16_t  uint_least16_t;
+typedef uint32_t  uint_least32_t;
+typedef uint64_t  uint_least64_t;
+
+// 7.18.1.3 Fastest minimum-width integer types
+typedef int8_t    int_fast8_t;
+typedef int16_t   int_fast16_t;
+typedef int32_t   int_fast32_t;
+typedef int64_t   int_fast64_t;
+typedef uint8_t   uint_fast8_t;
+typedef uint16_t  uint_fast16_t;
+typedef uint32_t  uint_fast32_t;
+typedef uint64_t  uint_fast64_t;
+
+// 7.18.1.4 Integer types capable of holding object pointers
+#ifdef _WIN64 // [
+   typedef __int64           intptr_t;
+   typedef unsigned __int64  uintptr_t;
+#else // _WIN64 ][
+   typedef _W64 int               intptr_t;
+   typedef _W64 unsigned int      uintptr_t;
+#endif // _WIN64 ]
+
+// 7.18.1.5 Greatest-width integer types
+typedef int64_t   intmax_t;
+typedef uint64_t  uintmax_t;
+
+
+// 7.18.2 Limits of specified-width integer types
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
+
+// 7.18.2.1 Limits of exact-width integer types
+#define INT8_MIN     ((int8_t)_I8_MIN)
+#define INT8_MAX     _I8_MAX
+#define INT16_MIN    ((int16_t)_I16_MIN)
+#define INT16_MAX    _I16_MAX
+#define INT32_MIN    ((int32_t)_I32_MIN)
+#define INT32_MAX    _I32_MAX
+#define INT64_MIN    ((int64_t)_I64_MIN)
+#define INT64_MAX    _I64_MAX
+#define UINT8_MAX    _UI8_MAX
+#define UINT16_MAX   _UI16_MAX
+#define UINT32_MAX   _UI32_MAX
+#define UINT64_MAX   _UI64_MAX
+
+// 7.18.2.2 Limits of minimum-width integer types
+#define INT_LEAST8_MIN    INT8_MIN
+#define INT_LEAST8_MAX    INT8_MAX
+#define INT_LEAST16_MIN   INT16_MIN
+#define INT_LEAST16_MAX   INT16_MAX
+#define INT_LEAST32_MIN   INT32_MIN
+#define INT_LEAST32_MAX   INT32_MAX
+#define INT_LEAST64_MIN   INT64_MIN
+#define INT_LEAST64_MAX   INT64_MAX
+#define UINT_LEAST8_MAX   UINT8_MAX
+#define UINT_LEAST16_MAX  UINT16_MAX
+#define UINT_LEAST32_MAX  UINT32_MAX
+#define UINT_LEAST64_MAX  UINT64_MAX
+
+// 7.18.2.3 Limits of fastest minimum-width integer types
+#define INT_FAST8_MIN    INT8_MIN
+#define INT_FAST8_MAX    INT8_MAX
+#define INT_FAST16_MIN   INT16_MIN
+#define INT_FAST16_MAX   INT16_MAX
+#define INT_FAST32_MIN   INT32_MIN
+#define INT_FAST32_MAX   INT32_MAX
+#define INT_FAST64_MIN   INT64_MIN
+#define INT_FAST64_MAX   INT64_MAX
+#define UINT_FAST8_MAX   UINT8_MAX
+#define UINT_FAST16_MAX  UINT16_MAX
+#define UINT_FAST32_MAX  UINT32_MAX
+#define UINT_FAST64_MAX  UINT64_MAX
+
+// 7.18.2.4 Limits of integer types capable of holding object pointers
+#ifdef _WIN64 // [
+#  define INTPTR_MIN   INT64_MIN
+#  define INTPTR_MAX   INT64_MAX
+#  define UINTPTR_MAX  UINT64_MAX
+#else // _WIN64 ][
+#  define INTPTR_MIN   INT32_MIN
+#  define INTPTR_MAX   INT32_MAX
+#  define UINTPTR_MAX  UINT32_MAX
+#endif // _WIN64 ]
+
+// 7.18.2.5 Limits of greatest-width integer types
+#define INTMAX_MIN   INT64_MIN
+#define INTMAX_MAX   INT64_MAX
+#define UINTMAX_MAX  UINT64_MAX
+
+// 7.18.3 Limits of other integer types
+
+#ifdef _WIN64 // [
+#  define PTRDIFF_MIN  _I64_MIN
+#  define PTRDIFF_MAX  _I64_MAX
+#else  // _WIN64 ][
+#  define PTRDIFF_MIN  _I32_MIN
+#  define PTRDIFF_MAX  _I32_MAX
+#endif  // _WIN64 ]
+
+#define SIG_ATOMIC_MIN  INT_MIN
+#define SIG_ATOMIC_MAX  INT_MAX
+
+#ifndef SIZE_MAX // [
+#  ifdef _WIN64 // [
+#     define SIZE_MAX  _UI64_MAX
+#  else // _WIN64 ][
+#     define SIZE_MAX  _UI32_MAX
+#  endif // _WIN64 ]
+#endif // SIZE_MAX ]
+
+// WCHAR_MIN and WCHAR_MAX are also defined in 
+#ifndef WCHAR_MIN // [
+#  define WCHAR_MIN  0
+#endif  // WCHAR_MIN ]
+#ifndef WCHAR_MAX // [
+#  define WCHAR_MAX  _UI16_MAX
+#endif  // WCHAR_MAX ]
+
+#define WINT_MIN  0
+#define WINT_MAX  _UI16_MAX
+
+#endif // __STDC_LIMIT_MACROS ]
+
+
+// 7.18.4 Limits of other integer types
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
+
+// 7.18.4.1 Macros for minimum-width integer constants
+
+#define INT8_C(val)  val##i8
+#define INT16_C(val) val##i16
+#define INT32_C(val) val##i32
+#define INT64_C(val) val##i64
+
+#define UINT8_C(val)  val##ui8
+#define UINT16_C(val) val##ui16
+#define UINT32_C(val) val##ui32
+#define UINT64_C(val) val##ui64
+
+// 7.18.4.2 Macros for greatest-width integer constants
+#define INTMAX_C   INT64_C
+#define UINTMAX_C  UINT64_C
+
+#endif // __STDC_CONSTANT_MACROS ]
+
+
+#endif // _MSC_STDINT_H_ ]
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
deleted file mode 100644
--- a/lib_pypy/_subprocess.py
+++ /dev/null
@@ -1,214 +0,0 @@
-"""
-Support routines for subprocess module.
-Currently, this extension module is only required when using the
-subprocess module on Windows.
-"""
-
-
-# Declare external Win32 functions
-
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
-
-GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
-
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessW
-_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
-
-# Now the _subprocess module implementation
-
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
-
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
-
-    def __int__(self):
-        return self.handle
-
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
-
-    def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
-
-    def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
-
-def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
-
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
-
-    if not res:
-        raise _WinError()
-
-    return _handle(read.value), _handle(write.value)
-
-def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
-
-def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
-
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
-
-    if not res:
-        raise _WinError()
-
-    return _handle(target.value)
-
-def CreateProcess(name, command_line, process_attr, thread_attr,
-                  inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
-    if startup_info is not None:
-        si.dwFlags = startup_info.dwFlags
-        si.wShowWindow = startup_info.wShowWindow
-        if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
-        if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
-        if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
-
-    pi = _PROCESS_INFORMATION()
-    flags |= CREATE_UNICODE_ENVIRONMENT
-
-    if env is not None:
-        envbuf = ""
-        for k, v in env.items():
-            envbuf += "%s=%s\0" % (k, v)
-        envbuf += '\0'
-    else:
-        envbuf = None
-
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
-
-    if not res:
-        raise _WinError()
-
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
-
-def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
-    if res < 0:
-        raise _WinError()
-
-    return res
-
-def GetExitCodeProcess(handle):
-    code = _c_int()
-
-    res = _GetExitCodeProcess(int(handle), _byref(code))
-
-    if not res:
-        raise _WinError()
-
-    return code.value
-
-def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
-
-    if not res:
-        raise _WinError()
-
-def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
-
-    if not res:
-        return None
-    else:
-        return res
-
-STD_INPUT_HANDLE = -10
-STD_OUTPUT_HANDLE = -11
-STD_ERROR_HANDLE = -12
-DUPLICATE_SAME_ACCESS = 2
-STARTF_USESTDHANDLES = 0x100
-STARTF_USESHOWWINDOW = 0x001
-SW_HIDE = 0
-INFINITE = 0xffffffff
-WAIT_OBJECT_0 = 0
-CREATE_NEW_CONSOLE = 0x010
-CREATE_NEW_PROCESS_GROUP = 0x200
-CREATE_UNICODE_ENVIRONMENT = 0x400
-STILL_ACTIVE = 259
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_winapi.py
@@ -0,0 +1,237 @@
+"""
+Support routines for subprocess module.
+Currently, this extension module is only required when using the
+subprocess module on Windows.
+"""
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
+
+# Declare external Win32 functions
+
+import ctypes
+
+_kernel32 = ctypes.WinDLL('kernel32')
+
+_CloseHandle = _kernel32.CloseHandle
+_CloseHandle.argtypes = [ctypes.c_int]
+_CloseHandle.restype = ctypes.c_int
+
+_CreatePipe = _kernel32.CreatePipe
+_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
+                        ctypes.c_void_p, ctypes.c_int]
+_CreatePipe.restype = ctypes.c_int
+
+_GetCurrentProcess = _kernel32.GetCurrentProcess
+_GetCurrentProcess.argtypes = []
+_GetCurrentProcess.restype = ctypes.c_int
+
+GetVersion = _kernel32.GetVersion
+GetVersion.argtypes = []
+GetVersion.restype = ctypes.c_int
+
+_DuplicateHandle = _kernel32.DuplicateHandle
+_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
+                             ctypes.POINTER(ctypes.c_int),
+                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
+_DuplicateHandle.restype = ctypes.c_int
+
+_WaitForSingleObject = _kernel32.WaitForSingleObject
+_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
+_WaitForSingleObject.restype = ctypes.c_int
+
+_GetExitCodeProcess = _kernel32.GetExitCodeProcess
+_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+_GetExitCodeProcess.restype = ctypes.c_int
+
+_TerminateProcess = _kernel32.TerminateProcess
+_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
+_TerminateProcess.restype = ctypes.c_int
+
+_GetStdHandle = _kernel32.GetStdHandle
+_GetStdHandle.argtypes = [ctypes.c_int]
+_GetStdHandle.restype = ctypes.c_int
+
+_GetModuleFileNameW = _kernel32.GetModuleFileNameW
+_GetModuleFileNameW.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_uint]
+_GetModuleFileNameW.restype = ctypes.c_int
+
+class _STARTUPINFO(ctypes.Structure):
+    _fields_ = [('cb',         ctypes.c_int),
+                ('lpReserved', ctypes.c_void_p),
+                ('lpDesktop',  ctypes.c_char_p),
+                ('lpTitle',    ctypes.c_char_p),
+                ('dwX',        ctypes.c_int),
+                ('dwY',        ctypes.c_int),
+                ('dwXSize',    ctypes.c_int),
+                ('dwYSize',    ctypes.c_int),
+                ('dwXCountChars', ctypes.c_int),
+                ('dwYCountChars', ctypes.c_int),
+                ("dwFillAttribute", ctypes.c_int),
+                ("dwFlags", ctypes.c_int),
+                ("wShowWindow", ctypes.c_short),
+                ("cbReserved2", ctypes.c_short),
+                ("lpReserved2", ctypes.c_void_p),
+                ("hStdInput", ctypes.c_int),
+                ("hStdOutput", ctypes.c_int),
+                ("hStdError", ctypes.c_int)
+                ]
+
+class _PROCESS_INFORMATION(ctypes.Structure):
+    _fields_ = [("hProcess", ctypes.c_int),
+                ("hThread", ctypes.c_int),
+                ("dwProcessID", ctypes.c_int),
+                ("dwThreadID", ctypes.c_int)]
+
+_CreateProcess = _kernel32.CreateProcessW
+_CreateProcess.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_void_p,
+                           ctypes.c_int, ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p,
+                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
+_CreateProcess.restype = ctypes.c_int
+
+del ctypes
+
+# Now the _winapi module implementation
+
+from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+
+class _handle:
+    def __init__(self, handle):
+        self.handle = handle
+
+    def __int__(self):
+        return self.handle
+
+    def __del__(self):
+        if self.handle is not None:
+            _CloseHandle(self.handle)
+
+    def Detach(self):
+        handle, self.handle = self.handle, None
+        return handle
+
+    def Close(self):
+        if self.handle not in (-1, None):
+            _CloseHandle(self.handle)
+            self.handle = None
+
+def CreatePipe(attributes, size):
+    read = _c_int()
+    write = _c_int()
+
+    res = _CreatePipe(_byref(read), _byref(write), None, size)
+
+    if not res:
+        raise _WinError()
+
+    return _handle(read.value), _handle(write.value)
+
+def GetCurrentProcess():
+    return _handle(_GetCurrentProcess())
+
+def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
+    target = _c_int()
+
+    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
+                           _byref(target),
+                           access, inherit, options)
+
+    if not res:
+        raise _WinError()
+
+    return _handle(target.value)
+
+def CreateProcess(name, command_line, process_attr, thread_attr,
+                  inherit, flags, env, start_dir, startup_info):
+    si = _STARTUPINFO()
+    if startup_info is not None:
+        si.dwFlags = startup_info.dwFlags
+        si.wShowWindow = startup_info.wShowWindow
+        if startup_info.hStdInput:
+            si.hStdInput = int(startup_info.hStdInput)
+        if startup_info.hStdOutput:
+            si.hStdOutput = int(startup_info.hStdOutput)
+        if startup_info.hStdError:
+            si.hStdError = int(startup_info.hStdError)
+
+    pi = _PROCESS_INFORMATION()
+    flags |= CREATE_UNICODE_ENVIRONMENT
+
+    if env is not None:
+        envbuf = ""
+        for k, v in env.items():
+            envbuf += "%s=%s\0" % (k, v)
+        envbuf += '\0'
+    else:
+        envbuf = None
+
+    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
+                        start_dir, _byref(si), _byref(pi))
+
+    if not res:
+        raise _WinError()
+
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+
+def WaitForSingleObject(handle, milliseconds):
+    res = _WaitForSingleObject(int(handle), milliseconds)
+
+    if res < 0:
+        raise _WinError()
+
+    return res
+
+def GetExitCodeProcess(handle):
+    code = _c_int()
+
+    res = _GetExitCodeProcess(int(handle), _byref(code))
+
+    if not res:
+        raise _WinError()
+
+    return code.value
+
+def TerminateProcess(handle, exitcode):
+    res = _TerminateProcess(int(handle), exitcode)
+
+    if not res:
+        raise _WinError()
+
+def GetStdHandle(stdhandle):
+    res = _GetStdHandle(stdhandle)
+
+    if not res:
+        return None
+    else:
+        return res
+
+def CloseHandle(handle):
+    res = _CloseHandle(handle)
+
+    if not res:
+        raise _WinError()
+
+def GetModuleFileName(module):
+    buf = ctypes.create_unicode_buffer(_MAX_PATH)
+    res = _GetModuleFileNameW(module, buf, _MAX_PATH)
+
+    if not res:
+        raise _WinError()
+    return buf.value
+
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+STD_ERROR_HANDLE = -12
+DUPLICATE_SAME_ACCESS = 2
+STARTF_USESTDHANDLES = 0x100
+STARTF_USESHOWWINDOW = 0x001
+SW_HIDE = 0
+INFINITE = 0xffffffff
+WAIT_OBJECT_0 = 0
+WAIT_TIMEOUT = 0x102
+CREATE_NEW_CONSOLE = 0x010
+CREATE_NEW_PROCESS_GROUP = 0x200
+CREATE_UNICODE_ENVIRONMENT = 0x400
+STILL_ACTIVE = 259
+_MAX_PATH = 260
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -35,8 +35,11 @@
                                "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError("Unsupported type: %r.  Please file a bug "
-                                   "if you think it should be." % (commontype,))
+                raise api.FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
             result, quals = parser.parse_type_and_quals(cdecl)   # recursive
 
         assert isinstance(result, model.BaseTypeByIdentity)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1193,8 +1193,7 @@
         elif flag == 'S':
             return False
         else:
-            return (self.lookup(w_obj, '__getitem__') is not None and
-                    self.lookup(w_obj, '__getslice__') is None)
+            return self.lookup(w_obj, '__getitem__') is not None
 
     # The code below only works
     # for the simple case (new-style instance).
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -117,8 +117,17 @@
         else:
             compare = space.lt
             jitdriver = min_jitdriver
+        any_kwds = bool(args.keywords)
         args_w = args.arguments_w
         if len(args_w) > 1:
+            if unroll and len(args_w) == 2 and not any_kwds:
+                # a fast path for the common case, useful for interpreted
+                # mode and to reduce the length of the jit trace
+                w0, w1 = args_w
+                if space.is_true(compare(w1, w0)):
+                    return w1
+                else:
+                    return w0
             w_sequence = space.newtuple(args_w)
         elif len(args_w):
             w_sequence = args_w[0]
@@ -127,8 +136,8 @@
                         "%s() expects at least one argument",
                         implementation_of)
         w_key = None
-        kwds = args.keywords
-        if kwds:
+        if any_kwds:
+            kwds = args.keywords
             if kwds[0] == "key" and len(kwds) == 1:
                 w_key = args.keywords_w[0]
             else:
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -585,6 +585,11 @@
         assert min([1, 2, 3]) == 1
         raises(TypeError, min, 1, 2, bar=2)
         raises(TypeError, min, 1, 2, key=lambda x: x, bar=2)
+        assert type(min(1, 1.0)) is int
+        assert type(min(1.0, 1)) is float
+        assert type(min(1, 1.0, 1L)) is int
+        assert type(min(1.0, 1L, 1)) is float
+        assert type(min(1L, 1, 1.0)) is long
 
     def test_max(self):
         assert max(1, 2) == 2
@@ -592,3 +597,8 @@
         assert max([1, 2, 3]) == 3
         raises(TypeError, max, 1, 2, bar=2)
         raises(TypeError, max, 1, 2, key=lambda x: x, bar=2)
+        assert type(max(1, 1.0)) is int
+        assert type(max(1.0, 1)) is float
+        assert type(max(1, 1.0, 1L)) is int
+        assert type(max(1.0, 1L, 1)) is float
+        assert type(max(1L, 1, 1.0)) is long
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -196,9 +196,13 @@
                 if is_getattr and attr == '__dict__':
                     return self.full_dict_copy()
                 if is_getattr and attr == '__class__':
-                    return self.space.type(self)
+                    # used to be space.type(self).  But HAAAAAACK!
+                    # That makes help() behave correctly.  I couldn't
+                    # find a more reasonable way.  Urgh.
+                    from pypy.interpreter.module import Module
+                    return self.space.gettypeobject(Module.typedef)
                 if is_getattr and attr == '__name__':
-                    return self.descr_repr()
+                    return self.space.wrap("%s.lib" % self.libname)
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1039,8 +1039,8 @@
         assert MYFOO == 42
         assert hasattr(lib, '__dict__')
         assert lib.__all__ == ['MYFOO', 'mybar']   # but not 'myvar'
-        assert lib.__name__ == repr(lib)
-        assert lib.__class__ is type(lib)
+        assert lib.__name__ == '_CFFI_test_import_from_lib.lib'
+        assert lib.__class__ is type(sys)   # !! hack for help()
 
     def test_macro_var_callback(self):
         ffi, lib = self.prepare(
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -1,4 +1,5 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.annlowlevel import llhelper
 from pypy.module.cpyext.pyobject import PyObject, make_ref
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct,
     PyObjectFields)
@@ -16,6 +17,23 @@
      ('TimeType', PyTypeObjectPtr),
      ('DeltaType', PyTypeObjectPtr),
      ('TZInfoType', PyTypeObjectPtr),
+
+     ('Date_FromDate', lltype.Ptr(lltype.FuncType(
+         [rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr],
+         PyObject))),
+     ('Time_FromTime', lltype.Ptr(lltype.FuncType(
+         [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+          PyObject, PyTypeObjectPtr],
+         PyObject))),
+     ('DateTime_FromDateAndTime', lltype.Ptr(lltype.FuncType(
+         [rffi.INT_real, rffi.INT_real, rffi.INT_real,
+          rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+          PyObject, PyTypeObjectPtr],
+         PyObject))),
+     ('Delta_FromDelta', lltype.Ptr(lltype.FuncType(
+         [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+          PyTypeObjectPtr],
+         PyObject))),
      ))
 
 @cpython_api([], lltype.Ptr(PyDateTime_CAPI))
@@ -45,6 +63,19 @@
     datetimeAPI.c_TZInfoType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
+    datetimeAPI.c_Date_FromDate = llhelper(
+        _PyDate_FromDate.api_func.functype,
+        _PyDate_FromDate.api_func.get_wrapper(space))
+    datetimeAPI.c_Time_FromTime = llhelper(
+        _PyTime_FromTime.api_func.functype,
+        _PyTime_FromTime.api_func.get_wrapper(space))
+    datetimeAPI.c_DateTime_FromDateAndTime = llhelper(
+        _PyDateTime_FromDateAndTime.api_func.functype,
+        _PyDateTime_FromDateAndTime.api_func.get_wrapper(space))
+    datetimeAPI.c_Delta_FromDelta = llhelper(
+        _PyDelta_FromDelta.api_func.functype,
+        _PyDelta_FromDelta.api_func.get_wrapper(space))
+
     return datetimeAPI
 
 PyDateTime_DateStruct = lltype.ForwardReference()
@@ -94,36 +125,40 @@
 make_check_function("PyDelta_Check", "timedelta")
 make_check_function("PyTZInfo_Check", "tzinfo")
 
-# Constructors
+# Constructors. They are better used as macros.
 
- at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject)
-def PyDate_FromDate(space, year, month, day):
+ at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr],
+             PyObject)
+def _PyDate_FromDate(space, year, month, day, w_type):
     """Return a datetime.date object with the specified year, month and day.
     """
     year = rffi.cast(lltype.Signed, year)
     month = rffi.cast(lltype.Signed, month)
     day = rffi.cast(lltype.Signed, day)
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    return space.call_method(
-        w_datetime, "date",
+    return space.call_function(
+        w_type,
         space.wrap(year), space.wrap(month), space.wrap(day))
 
- at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject)
-def PyTime_FromTime(space, hour, minute, second, usecond):
+ at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+              PyObject, PyTypeObjectPtr], PyObject)
+def _PyTime_FromTime(space, hour, minute, second, usecond, w_tzinfo, w_type):
     """Return a ``datetime.time`` object with the specified hour, minute, second and
     microsecond."""
     hour = rffi.cast(lltype.Signed, hour)
     minute = rffi.cast(lltype.Signed, minute)
     second = rffi.cast(lltype.Signed, second)
     usecond = rffi.cast(lltype.Signed, usecond)
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    return space.call_method(
-        w_datetime, "time",
+    return space.call_function(
+        w_type,
         space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond))
+        space.wrap(usecond), w_tzinfo)
 
- at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject)
-def PyDateTime_FromDateAndTime(space, year, month, day, hour, minute, second, usecond):
+ at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real,
+              rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+              PyObject, PyTypeObjectPtr], PyObject)
+def _PyDateTime_FromDateAndTime(space, year, month, day,
+                                hour, minute, second, usecond,
+                                w_tzinfo, w_type):
     """Return a datetime.datetime object with the specified year, month, day, hour,
     minute, second and microsecond.
     """
@@ -134,12 +169,11 @@
     minute = rffi.cast(lltype.Signed, minute)
     second = rffi.cast(lltype.Signed, second)
     usecond = rffi.cast(lltype.Signed, usecond)
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    return space.call_method(
-        w_datetime, "datetime",
+    return space.call_function(
+        w_type,
         space.wrap(year), space.wrap(month), space.wrap(day),
         space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond))
+        space.wrap(usecond), w_tzinfo)
 
 @cpython_api([PyObject], PyObject)
 def PyDateTime_FromTimestamp(space, w_args):
@@ -161,8 +195,10 @@
     w_method = space.getattr(w_type, space.wrap("fromtimestamp"))
     return space.call(w_method, w_args)
 
- at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject)
-def PyDelta_FromDSU(space, days, seconds, useconds):
+ at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
+              PyTypeObjectPtr],
+             PyObject)
+def _PyDelta_FromDelta(space, days, seconds, useconds, normalize, w_type):
     """Return a datetime.timedelta object representing the given number of days,
     seconds and microseconds.  Normalization is performed so that the resulting
     number of microseconds and seconds lie in the ranges documented for
@@ -171,9 +207,8 @@
     days = rffi.cast(lltype.Signed, days)
     seconds = rffi.cast(lltype.Signed, seconds)
     useconds = rffi.cast(lltype.Signed, useconds)
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    return space.call_method(
-        w_datetime, "timedelta",
+    return space.call_function(
+        w_type,
         space.wrap(days), space.wrap(seconds), space.wrap(useconds))
 
 # Accessors
diff --git a/pypy/module/cpyext/include/datetime.h b/pypy/module/cpyext/include/datetime.h
--- a/pypy/module/cpyext/include/datetime.h
+++ b/pypy/module/cpyext/include/datetime.h
@@ -12,6 +12,13 @@
     PyTypeObject *TimeType;
     PyTypeObject *DeltaType;
     PyTypeObject *TZInfoType;
+
+    /* constructors */
+    PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*);
+    PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int,
+        PyObject*, PyTypeObject*);
+    PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*);
+    PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*);
 } PyDateTime_CAPI;
 
 PyAPI_DATA(PyDateTime_CAPI*) PyDateTimeAPI;
@@ -41,6 +48,22 @@
     PyObject_HEAD
 } PyDateTime_TZInfo;
 
+/* Macros for accessing constructors in a simplified fashion. */
+#define PyDate_FromDate(year, month, day) \
+    PyDateTimeAPI->Date_FromDate(year, month, day, PyDateTimeAPI->DateType)
+
+#define PyDateTime_FromDateAndTime(year, month, day, hour, min, sec, usec) \
+    PyDateTimeAPI->DateTime_FromDateAndTime(year, month, day, hour, \
+        min, sec, usec, Py_None, PyDateTimeAPI->DateTimeType)
+
+#define PyTime_FromTime(hour, minute, second, usecond) \
+    PyDateTimeAPI->Time_FromTime(hour, minute, second, usecond, \
+        Py_None, PyDateTimeAPI->TimeType)
+
+#define PyDelta_FromDSU(days, seconds, useconds) \
+    PyDateTimeAPI->Delta_FromDelta(days, seconds, useconds, 1, \
+        PyDateTimeAPI->DeltaType)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/src/unicodeobject.c b/pypy/module/cpyext/src/unicodeobject.c
--- a/pypy/module/cpyext/src/unicodeobject.c
+++ b/pypy/module/cpyext/src/unicodeobject.c
@@ -6,9 +6,6 @@
 #define Py_ISDIGIT isdigit
 #define Py_ISALPHA isalpha
 
-#define PyObject_Malloc malloc
-#define PyObject_Free free
-
 static void
 makefmt(char *fmt, int longflag, int longlongflag, int size_tflag,
         int zeropad, int width, int precision, char c)
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -4,7 +4,8 @@
 
 class TestDatetime(BaseApiTest):
     def test_date(self, space, api):
-        w_date = api.PyDate_FromDate(2010, 06, 03)
+        date_api = api._PyDateTime_Import()
+        w_date = api._PyDate_FromDate(2010, 06, 03, date_api.c_DateType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03'
 
         assert api.PyDate_Check(w_date)
@@ -15,7 +16,9 @@
         assert api.PyDateTime_GET_DAY(w_date) == 3
 
     def test_time(self, space, api):
-        w_time = api.PyTime_FromTime(23, 15, 40, 123456)
+        date_api = api._PyDateTime_Import()
+        w_time = api._PyTime_FromTime(23, 15, 40, 123456,
+                                      space.w_None, date_api.c_TimeType)
         assert space.unwrap(space.str(w_time)) == '23:15:40.123456'
 
         assert api.PyTime_Check(w_time)
@@ -27,8 +30,10 @@
         assert api.PyDateTime_TIME_GET_MICROSECOND(w_time) == 123456
 
     def test_datetime(self, space, api):
-        w_date = api.PyDateTime_FromDateAndTime(
-            2010, 06, 03, 23, 15, 40, 123456)
+        date_api = api._PyDateTime_Import()
+        w_date = api._PyDateTime_FromDateAndTime(
+            2010, 06, 03, 23, 15, 40, 123456,
+            space.w_None, date_api.c_DateTimeType)
         assert space.unwrap(space.str(w_date)) == '2010-06-03 23:15:40.123456'
 
         assert api.PyDateTime_Check(w_date)
@@ -45,6 +50,7 @@
         assert api.PyDateTime_DATE_GET_MICROSECOND(w_date) == 123456
 
     def test_delta(self, space, api):
+        date_api = api._PyDateTime_Import()
         w_delta = space.appexec(
             [space.wrap(3), space.wrap(15)], """(days, seconds):
             from datetime import timedelta
@@ -53,7 +59,7 @@
         assert api.PyDelta_Check(w_delta)
         assert api.PyDelta_CheckExact(w_delta)
 
-        w_delta = api.PyDelta_FromDSU(10, 20, 30)
+        w_delta = api._PyDelta_FromDelta(10, 20, 30, True, date_api.c_DeltaType)
         assert api.PyDelta_Check(w_delta)
         assert api.PyDelta_CheckExact(w_delta)
 
@@ -118,6 +124,31 @@
                                       datetime.tzinfo)
         module.clear_types()
 
+    def test_constructors(self):
+        module = self.import_extension('foo', [
+            ("new_date", "METH_NOARGS",

From pypy.commits at gmail.com  Mon Jun 20 18:16:18 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Mon, 20 Jun 2016 15:16:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Merged in marky1991/pypy_new/py3k (pull
 request #454)
Message-ID: <57686b32.c255c20a.588fa.0127@mx.google.com>

Author: Philip Jenvey 
Branch: py3k
Changeset: r85294:ba36ebd247e1
Date: 2016-06-20 15:15 -0700
http://bitbucket.org/pypy/pypy/changeset/ba36ebd247e1/

Log:	Merged in marky1991/pypy_new/py3k (pull request #454)

	Py3k: Update Fallback Code in Time to Match CPython

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -418,7 +418,7 @@
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
     RegrTest('test_threadsignals.py', usemodules="thread"),
-    RegrTest('test_time.py', core=True, usemodules="struct"),
+    RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),
     RegrTest('test_timeit.py'),
     RegrTest('test_timeout.py'),
     RegrTest('test_tk.py'),
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -3,8 +3,10 @@
 from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.rtime import win_perf_counter
+from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat
+from rpython.rlib.rtime import (win_perf_counter, TIMEB, c_ftime,
+                                GETTIMEOFDAY_NO_TZ, TIMEVAL,
+                                HAVE_GETTIMEOFDAY, HAVE_FTIME)
 from rpython.rlib import rposix, rtime
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 import math
@@ -81,16 +83,6 @@
         [rffi.VOIDP],
         rffi.ULONGLONG, compilation_info=eci)
 
-    from rpython.rlib.rdynload import GetModuleHandle, dlsym
-    hKernel32 = GetModuleHandle("KERNEL32")
-    try:
-        _GetTickCount64_handle = dlsym(hKernel32, 'GetTickCount64')
-        def _GetTickCount64():
-            return pypy_GetTickCount64(_GetTickCount64_handle)
-    except KeyError:
-        _GetTickCount64_handle = lltype.nullptr(rffi.VOIDP.TO)
-
-    HAS_GETTICKCOUNT64 = _GetTickCount64_handle != lltype.nullptr(rffi.VOIDP.TO)
     class GlobalState:
         def __init__(self):
             self.init()
@@ -126,13 +118,31 @@
         def get_interrupt_event(self):
             return globalState.interrupt_event
 
-    # XXX: Can I just use one of the state classes above?
-    # I don't really get why an instance is better than a plain module
-    # attr, but following advice from armin
     class TimeState(object):
+        GetTickCount64_handle = lltype.nullptr(rffi.VOIDP.TO)
         def __init__(self):
             self.n_overflow = 0
             self.last_ticks = 0
+
+        def check_GetTickCount64(self, *args):
+            if (self.GetTickCount64_handle !=
+                lltype.nullptr(rffi.VOIDP.TO)):
+                return True
+            from rpython.rlib.rdynload import GetModuleHandle, dlsym
+            hKernel32 = GetModuleHandle("KERNEL32")
+            try:
+                GetTickCount64_handle = dlsym(hKernel32, 'GetTickCount64')
+            except KeyError:
+                return False
+            self.GetTickCount64_handle = GetTickCount64_handle
+            return True
+
+        def GetTickCount64(self, *args):
+            assert (self.GetTickCount64_handle !=
+                    lltype.nullptr(rffi.VOIDP.TO))
+            return pypy_GetTickCount64(
+                self.GetTickCount64_handle, *args)
+
     time_state = TimeState()
 
 _includes = ["time.h"]
@@ -182,13 +192,6 @@
         ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
-    # TODO: Figure out how to implement this...
-    CConfig.ULARGE_INTEGER = platform.Struct("struct ULARGE_INTEGER", [
-        ("tm_sec", rffi.INT),
-        ("tm_min", rffi.INT), ("tm_hour", rffi.INT), ("tm_mday", rffi.INT),
-        ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
-        ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
-
 if _MACOSX:
     CConfig.TIMEBASE_INFO = platform.Struct("struct mach_timebase_info", [
         ("numer", rffi.UINT),
@@ -228,35 +231,75 @@
 tm = cConfig.tm
 glob_buf = lltype.malloc(tm, flavor='raw', zero=True, immortal=True)
 
-if cConfig.has_gettimeofday:
-    c_gettimeofday = external('gettimeofday',
-                              [cConfig.timeval, rffi.VOIDP], rffi.INT)
-    if _WIN:
-        GetSystemTimeAsFileTime = external('GetSystemTimeAsFileTime',
-                                           [rwin32.FILETIME],
-                                           lltype.VOID)
-        def gettimeofday(space, w_info=None):
-            with lltype.scoped_alloc(rwin32.FILETIME) as system_time:
-                GetSystemTimeAsFileTime(system_time)
-                # XXX:
-                #seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
-                # XXX: w_info
-            return space.w_None
-    else:
-        def gettimeofday(space, w_info=None):
-            with lltype.scoped_alloc(CConfig.timeval) as timeval:
-                ret = c_gettimeofday(timeval, rffi.NULL)
-                if ret != 0:
-                    raise exception_from_saved_errno(space, space.w_OSError)
+if _WIN:
+    _GetSystemTimeAsFileTime = rwin32.winexternal('GetSystemTimeAsFileTime',
+                                                  [lltype.Ptr(rwin32.FILETIME)],
+                                                  lltype.Void)
+    LPDWORD = rwin32.LPDWORD
+    _GetSystemTimeAdjustment = rwin32.winexternal(
+                                            'GetSystemTimeAdjustment',
+                                            [LPDWORD, LPDWORD, rwin32.LPBOOL], 
+                                            rffi.INT)
+    def gettimeofday(space, w_info=None):
+        with lltype.scoped_alloc(rwin32.FILETIME) as system_time:
+            _GetSystemTimeAsFileTime(system_time)
+            quad_part = (system_time.c_dwLowDateTime |
+                         (r_ulonglong(system_time.c_dwHighDateTime) << 32))
+            # 11,644,473,600,000,000: number of microseconds between
+            # the 1st january 1601 and the 1st january 1970 (369 years + 80 leap
+            # days).
 
+            # We can't use that big number when translating for
+            # 32-bit system (which windows always is currently)
+            # XXX: Need to come up with a better solution
+            offset = (r_ulonglong(16384) * r_ulonglong(27) * r_ulonglong(390625)
+                     * r_ulonglong(79) * r_ulonglong(853))
+            microseconds = quad_part / 10 - offset
+            tv_sec = microseconds / 1000000
+            tv_usec = microseconds % 1000000
+            if w_info:
+                with lltype.scoped_alloc(LPDWORD.TO, 1) as time_adjustment, \
+                     lltype.scoped_alloc(LPDWORD.TO, 1) as time_increment, \
+                     lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
+                    _GetSystemTimeAdjustment(time_adjustment, time_increment,
+                                             is_time_adjustment_disabled)
+                    
+                    _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
+                             time_increment[0] * 1e-7, False, True)
+            return space.wrap(tv_sec + tv_usec * 1e-6)
+else:
+    if HAVE_GETTIMEOFDAY:
+        if GETTIMEOFDAY_NO_TZ:
+            c_gettimeofday = external('gettimeofday',
+                                      [lltype.Ptr(TIMEVAL)], rffi.INT)
+        else:
+            c_gettimeofday = external('gettimeofday',
+                                      [lltype.Ptr(TIMEVAL), rffi.VOIDP], rffi.INT)
+    def gettimeofday(space, w_info=None):
+        if HAVE_GETTIMEOFDAY:
+            with lltype.scoped_alloc(TIMEVAL) as timeval:
+                if GETTIMEOFDAY_NO_TZ:
+                    errcode = c_gettimeofday(timeval)
+                else:
+                    void = lltype.nullptr(rffi.VOIDP.TO)
+                    errcode = c_gettimeofday(timeval, void)
+                if rffi.cast(rffi.LONG, errcode) == 0:
+                    if w_info is not None:
+                        _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
+                    return space.wrap(timeval.c_tv_sec + timeval.c_tv_usec * 1e-6)
+        if HAVE_FTIME:
+            with lltype.scoped_alloc(TIMEB) as t:
+                c_ftime(t)
+                result = (float(intmask(t.c_time)) +
+                          float(intmask(t.c_millitm)) * 0.001)
                 if w_info is not None:
-                    _setinfo(space, w_info,
-                             "gettimeofday()", 1e-6, False, True)
-
-                seconds = float(timeval.tv_sec) + timeval.tv_usec * 1e-6
-            return space.wrap(seconds)
-
-
+                    _setinfo(space, w_info, "ftime()", 1e-3,
+                             False, True) 
+            return space.wrap(result)
+        else:
+            if w_info:
+                _setinfo(space, w_info, "time()", 1.0, False, True)
+            return space.wrap(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
 
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
@@ -584,22 +627,8 @@
                         _setinfo(space, w_info, "clock_gettime(CLOCK_REALTIME)",
                                  res, False, True)
                 return space.wrap(_timespec_to_seconds(timespec))
-
-    # XXX: rewrite the final fallback into gettimeofday w/ windows
-    # GetSystemTimeAsFileTime() support
-    secs = pytime.time()
-    if w_info is not None:
-        # XXX: time.time delegates to the host python's time.time
-        # (rtime.time) so duplicate its internals for now
-        if rtime.HAVE_GETTIMEOFDAY:
-            implementation = "gettimeofday()"
-            resolution = 1e-6
-        else: # assume using ftime(3)
-            implementation = "ftime()"
-            resolution = 1e-3
-        _setinfo(space, w_info, implementation, resolution, False, True)
-    return space.wrap(secs)
-
+    else:
+        return gettimeofday(space, w_info)
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -800,20 +829,13 @@
 
 
 if _WIN:
-    # untested so far
     _GetTickCount = rwin32.winexternal('GetTickCount', [], rwin32.DWORD)
-    LPDWORD = rwin32.LPDWORD
-    _GetSystemTimeAdjustment = rwin32.winexternal(
-                                            'GetSystemTimeAdjustment',
-                                            [LPDWORD, LPDWORD, rwin32.LPBOOL],
-                                            rffi.INT)
     def monotonic(space, w_info=None):
         result = 0
+        HAS_GETTICKCOUNT64 = time_state.check_GetTickCount64()
         if HAS_GETTICKCOUNT64:
-            print('has count64'.encode('ascii'))
-            result = _GetTickCount64() * 1e-3
+            result = time_state.GetTickCount64() * 1e-3
         else:
-            print("nocount64")
             ticks = _GetTickCount()
             if ticks < time_state.last_ticks:
                 time_state.n_overflow += 1
@@ -828,11 +850,9 @@
             else:
                 implementation = "GetTickCount()"
             resolution = 1e-7
-            print("creating a thing".encode("ascii"))
             with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_adjustment, \
                  lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as time_increment, \
                  lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as is_time_adjustment_disabled:
-                print("CREATED".encode("ascii"))
                 ok = _GetSystemTimeAdjustment(time_adjustment,
                                               time_increment,
                                               is_time_adjustment_disabled)
@@ -841,7 +861,6 @@
                     raise wrap_windowserror(space,
                         rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
                 resolution = resolution * time_increment[0]
-            print("out of with".encode("ascii"))
             _setinfo(space, w_info, implementation, resolution, True, False)
         return space.wrap(result)
 
@@ -981,7 +1000,7 @@
         Return the CPU time or real time since the start of the process or since
         the first call to clock().  This has as much precision as the system
         records."""
-        return space.wrap(win_perf_counter(space, w_info=w_info))
+        return space.wrap(perf_counter(space, w_info=w_info))
 
 else:
     _clock = external('clock', [], clock_t)
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -275,6 +275,7 @@
         # input to [w]strftime is not kosher.
         if os.name == 'nt':
             raises(ValueError, time.strftime, '%f')
+            return
         elif sys.platform == 'darwin' or 'bsd' in sys.platform:
             # darwin strips % of unknown format codes
             # http://bugs.python.org/issue9811

From pypy.commits at gmail.com  Mon Jun 20 18:16:35 2016
From: pypy.commits at gmail.com (marky1991)
Date: Mon, 20 Jun 2016 15:16:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Trying something. (Working on my linux box
 so as to avoid work in cmd.exe)
Message-ID: <57686b43.c5301c0a.734dd.3d8a@mx.google.com>

Author: Mark Young 
Branch: py3k
Changeset: r85268:3fa9246938e2
Date: 2016-05-22 14:43 -0400
http://bitbucket.org/pypy/pypy/changeset/3fa9246938e2/

Log:	Trying something. (Working on my linux box so as to avoid work in
	cmd.exe)

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -732,7 +732,7 @@
     LPDWORD = rwin32.LPDWORD
     _GetSystemTimeAdjustment = rwin32.winexternal(
                                             'GetSystemTimeAdjustment',
-                                            [LPDWORD, LPDWORD, rffi.INTP], 
+                                            [LPDWORD, LPDWORD, rffi.LPBOOL], 
                                             rffi.INT)
 
     def monotonic(space, w_info=None):
@@ -758,7 +758,7 @@
             resolution = 1e-7
             with lltype.scoped_alloc(rwin32.LPDWORD) as time_adjustment, \
                  lltype.scoped_alloc(rwin32.LPDWORD) as time_increment, \
-                 lltype.scoped_alloc(rwin32.FILETIME) as is_time_adjustment_disabled:
+                 lltype.scoped_alloc(rwin32.LPBOOL) as is_time_adjustment_disabled:
                 ok = _GetSystemTimeAdjustment(time_adjustment,
                                               time_increment,
                                               is_time_adjustment_disabled)
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -46,6 +46,7 @@
         LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP)
         LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP)
         LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.UINTP)
+        LPBOOL = rffi_platform.SimpleType("LPBOOL", rffi.LONGP)
         SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T)
         ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG)
 

From pypy.commits at gmail.com  Mon Jun 20 21:24:00 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Mon, 20 Jun 2016 18:24:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make test_datetime.py
 CPython-compatible (with pytest -A).
Message-ID: <57689730.89acc20a.65989.0a53@mx.google.com>

Author: Devin Jeanpierre 
Branch: 
Changeset: r85295:0325ae17502b
Date: 2016-06-20 18:23 -0700
http://bitbucket.org/pypy/pypy/changeset/0325ae17502b/

Log:	Make test_datetime.py CPython-compatible (with pytest -A).

diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -142,7 +142,7 @@
                     2000, 6, 6, 6, 6, 6, 6, Py_None,
                     PyDateTimeAPI->DateTimeType);
              """),
-        ])
+        ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.new_date() == datetime.date(2000, 6, 6)
         assert module.new_time() == datetime.time(6, 6, 6, 6)
@@ -241,6 +241,9 @@
                  PyObject* obj = PyDelta_FromDSU(6, 6, 6);
                  PyDateTime_Delta* delta = (PyDateTime_Delta*)obj;
 
+#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000
+                 // These macros are only defined in CPython 3.x and PyPy.
+                 // See: http://bugs.python.org/issue13727
                  PyDateTime_DELTA_GET_DAYS(obj);
                  PyDateTime_DELTA_GET_DAYS(delta);
 
@@ -249,10 +252,10 @@
 
                  PyDateTime_DELTA_GET_MICROSECONDS(obj);
                  PyDateTime_DELTA_GET_MICROSECONDS(delta);
-
+#endif
                  return obj;
              """),
-            ])
+            ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.test_date_macros() == datetime.date(2000, 6, 6)
         assert module.test_datetime_macros() == datetime.datetime(

From pypy.commits at gmail.com  Mon Jun 20 21:59:11 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Mon, 20 Jun 2016 18:59:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make test_typeobject.py CPython
 compatible (with pytest -A).
Message-ID: <57689f6f.a90bc30a.19602.4b72@mx.google.com>

Author: Devin Jeanpierre 
Branch: 
Changeset: r85296:556c43cacee6
Date: 2016-06-20 18:31 -0700
http://bitbucket.org/pypy/pypy/changeset/556c43cacee6/

Log:	Make test_typeobject.py CPython compatible (with pytest -A).

	The comment previously said the size in CPython increased by 6, but
	that isn't true on my system, where it increases by 16. I suspect it
	depends on platform or is otherwise not worth staying consistent
	with.

	(And it isn't as if we tested it before... :])

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -897,7 +897,6 @@
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
-        # on cpython, the size changes (6 bytes added)
         module = self.import_module(name='foo')
         size = module.size_of_instances(module.fooType)
         class f1(object):
@@ -907,7 +906,11 @@
         class bar(f1, f2):
             pass
         assert bar.__base__ is f2
-        assert module.size_of_instances(bar) == size
+        # On cpython, the size changes.
+        if '__pypy__' in sys.builtin_module_names:
+            assert module.size_of_instances(bar) == size
+        else:
+            assert module.size_of_instances(bar) >= size
 
     def test_app_cant_subclass_two_types(self):
         module = self.import_module(name='foo')

From pypy.commits at gmail.com  Mon Jun 20 21:59:13 2016
From: pypy.commits at gmail.com (devin.jeanpierre)
Date: Mon, 20 Jun 2016 18:59:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make test_bytearrayobject.py
 CPython-compatible (with pytest -A).
Message-ID: <57689f71.89acc20a.65989.119e@mx.google.com>

Author: Devin Jeanpierre 
Branch: 
Changeset: r85297:cdcba39cd359
Date: 2016-06-20 18:40 -0700
http://bitbucket.org/pypy/pypy/changeset/cdcba39cd359/

Log:	Make test_bytearrayobject.py CPython-compatible (with pytest -A).

	The test asserted that PyByteArray_AsString would raise on CPython,
	but this is not documented to happen. In fact it seems like it just
	does potentially arbitrary memory unsafety if asserts are disabled.

	Unsettling.

diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -1,5 +1,7 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+import sys
+
 class AppTestStringObject(AppTestCpythonExtensionBase):
     def test_basic(self):
         module = self.import_extension('foo', [
@@ -157,7 +159,9 @@
              """)])
         assert module.bytearray_from_string("huheduwe") == "huhe"
         assert module.str_from_bytearray(bytearray('abc')) == 'abc'
-        raises(ValueError, module.str_from_bytearray, 4.0)
+        if '__pypy__' in sys.builtin_module_names:
+            # CPython only makes an assert.
+            raises(ValueError, module.str_from_bytearray, 4.0)
         ret = module.concat('abc', 'def')
         assert ret == 'abcdef'
         assert not isinstance(ret, str)

From pypy.commits at gmail.com  Mon Jun 20 23:16:32 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 20:16:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix sys imports
Message-ID: <5768b190.06a61c0a.3f464.2f49@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r85298:c58bfb70bcbb
Date: 2016-06-21 04:15 +0100
http://bitbucket.org/pypy/pypy/changeset/c58bfb70bcbb/

Log:	fix sys imports

diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -1,6 +1,5 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
-import sys
 
 class AppTestStringObject(AppTestCpythonExtensionBase):
     def test_basic(self):
@@ -33,7 +32,7 @@
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
                      printf("tp_basicsize==%ld\\n",
-                            (long)s->ob_type->tp_basicsize); 
+                            (long)s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -90,7 +89,7 @@
                 base = PyByteArray_FromStringAndSize("test", 10);
                 if (PyByteArray_GET_SIZE(base) != 10)
                     return PyLong_FromLong(-PyByteArray_GET_SIZE(base));
-                memcpy(PyByteArray_AS_STRING(base), "works", 6); 
+                memcpy(PyByteArray_AS_STRING(base), "works", 6);
                 Py_INCREF(base);
                 return base;
              """),
@@ -117,6 +116,7 @@
         assert s == 'test'
 
     def test_manipulations(self):
+        import sys
         module = self.import_extension('foo', [
             ("bytearray_from_string", "METH_VARARGS",
              '''
@@ -143,9 +143,9 @@
             ("concat", "METH_VARARGS",
              """
                 PyObject * ret, *right, *left;
-                PyObject *ba1, *ba2; 
+                PyObject *ba1, *ba2;
                 if (!PyArg_ParseTuple(args, "OO", &left, &right)) {
-                    return PyString_FromString("parse failed"); 
+                    return PyString_FromString("parse failed");
                 }
                 ba1 = PyByteArray_FromObject(left);
                 ba2 = PyByteArray_FromObject(right);
@@ -175,9 +175,9 @@
              PyObject *obj, *ba;
              int newsize, oldsize, ret;
              if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) {
-                 return PyString_FromString("parse failed"); 
+                 return PyString_FromString("parse failed");
              }
-             
+
              ba = PyByteArray_FromObject(obj);
              if (ba == NULL)
                  return NULL;
@@ -191,7 +191,7 @@
              {
                   printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize);
                   return NULL;
-             } 
+             }
              return ba;
              '''
             )])
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -5,8 +5,6 @@
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
-import sys
-
 class AppTestTypeObject(AppTestCpythonExtensionBase):
     def test_typeobject(self):
         import sys
@@ -897,6 +895,7 @@
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
+        import sys
         module = self.import_module(name='foo')
         size = module.size_of_instances(module.fooType)
         class f1(object):

From pypy.commits at gmail.com  Mon Jun 20 23:35:05 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 20:35:05 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: hg merge default
Message-ID: <5768b5e9.4a2e1c0a.2da35.33bb@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85299:5181a00cb020
Date: 2016-06-21 04:17 +0100
http://bitbucket.org/pypy/pypy/changeset/5181a00cb020/

Log:	hg merge default

diff too long, truncating to 2000 out of 2387 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,10 +1,17 @@
-=========================
+==========================
 What's new in PyPy2.7 5.3+
-=========================
+==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -22,3 +29,16 @@
 
 .. branch: incminimark-ll_assert
 .. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1210,8 +1210,6 @@
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
-            if space.is_w(w_type, space.w_str):
-                pto.c_tp_itemsize = 1
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
@@ -1520,7 +1518,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -6,7 +6,9 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref)
+    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
+    pyobj_has_w_obj)
+from pypy.objspace.std.bytesobject import W_BytesObject
 
 ##
 ## Implementation of PyStringObject
@@ -16,7 +18,7 @@
 ## -----------
 ##
 ## PyString_AsString() must return a (non-movable) pointer to the underlying
-## buffer, whereas pypy strings are movable.  C code may temporarily store
+## ob_sval, whereas pypy strings are movable.  C code may temporarily store
 ## this address and use it, as long as it owns a reference to the PyObject.
 ## There is no "release" function to specify that the pointer is not needed
 ## any more.
@@ -28,7 +30,7 @@
 ## --------
 ##
 ## PyStringObject contains two additional members: the ob_size and a pointer to a
-## char buffer; it may be NULL.
+## char ob_sval; it may be NULL.
 ##
 ## - A string allocated by pypy will be converted into a PyStringObject with a
 ##   NULL buffer.  The first time PyString_AsString() is called, memory is
@@ -41,6 +43,9 @@
 ##   the pypy string is created, and added to the global map of tracked
 ##   objects.  The buffer is then supposed to be immutable.
 ##
+##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
+##   there is no corresponding pypy object for the string
+##
 ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
 ##   similar object.
 ##
@@ -53,7 +58,7 @@
 PyStringObjectStruct = lltype.ForwardReference()
 PyStringObject = lltype.Ptr(PyStringObjectStruct)
 PyStringObjectFields = PyVarObjectFields + \
-    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP))
+    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char)))
 cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct)
 
 @bootstrap_function
@@ -69,44 +74,43 @@
 
 def new_empty_str(space, length):
     """
-    Allocate a PyStringObject and its buffer, but without a corresponding
-    interpreter object.  The buffer may be mutated, until string_realize() is
+    Allocate a PyStringObject and its ob_sval, but without a corresponding
+    interpreter object.  The ob_sval may be mutated, until string_realize() is
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_str.layout.typedef)
-    py_obj = typedescr.allocate(space, space.w_str)
+    py_obj = typedescr.allocate(space, space.w_str, length)
     py_str = rffi.cast(PyStringObject, py_obj)
-
-    buflen = length + 1
-    py_str.c_ob_size = length
-    py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+    py_str.c_ob_shash = -1
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
 def string_attach(space, py_obj, w_obj):
     """
-    Fills a newly allocated PyStringObject with the given string object. The
-    buffer must not be modified.
+    Copy RPython string object contents to a PyStringObject. The
+    c_ob_sval must not be modified.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    py_str.c_ob_size = len(space.str_w(w_obj))
-    py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
+    s = space.str_w(w_obj)
+    if py_str.c_ob_size  < len(s):
+        raise oefmt(space.w_ValueError,
+            "string_attach called on object with ob_size %d but trying to store %d",
+            py_str.c_ob_size, len(s)) 
+    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def string_realize(space, py_obj):
     """
-    Creates the string in the interpreter. The PyStringObject buffer must not
+    Creates the string in the interpreter. The PyStringObject ob_sval must not
     be modified after this call.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    if not py_str.c_buffer:
-        py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                    flavor='raw', zero=True)
-    s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
-    w_obj = space.wrap(s)
+    s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(W_BytesObject, w_type)
+    w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
@@ -116,9 +120,6 @@
 def string_dealloc(space, py_obj):
     """Frees allocated PyStringObject resources.
     """
-    py_str = rffi.cast(PyStringObject, py_obj)
-    if py_str.c_buffer:
-        lltype.free(py_str.c_buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -139,6 +140,9 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
+    return _PyString_AsString(space, ref)
+
+def _PyString_AsString(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         pass    # typecheck returned "ok" without forcing 'ref' at all
     elif not PyString_Check(space, ref):   # otherwise, use the alternate way
@@ -151,15 +155,23 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    return ref_str.c_buffer
+    if not pyobj_has_w_obj(ref):
+        # XXX Force the ref?
+        string_realize(space, ref)
+    return ref_str.c_ob_sval
+
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
+def PyString_AS_STRING(space, void_ref):
+    ref = rffi.cast(PyObject, void_ref)
+    # if no w_str is associated with this ref,
+    # return the c-level ptr as RW
+    if not pyobj_has_w_obj(ref):
+        py_str = rffi.cast(PyStringObject, ref)
+        return py_str.c_ob_sval
+    return _PyString_AsString(space, ref)
 
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
-def PyString_AsStringAndSize(space, ref, buffer, length):
+def PyString_AsStringAndSize(space, ref, data, length):
     if not PyString_Check(space, ref):
         from pypy.module.cpyext.unicodeobject import (
             PyUnicode_Check, _PyUnicode_AsDefaultEncodedString)
@@ -169,18 +181,16 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
+    if not pyobj_has_w_obj(ref):
+        # force the ref
+        string_realize(space, ref)
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    buffer[0] = ref_str.c_buffer
+    data[0] = ref_str.c_ob_sval
     if length:
         length[0] = ref_str.c_ob_size
     else:
         i = 0
-        while ref_str.c_buffer[i] != '\0':
+        while ref_str.c_ob_sval[i] != '\0':
             i += 1
         if i != ref_str.c_ob_size:
             raise oefmt(space.w_TypeError,
@@ -209,10 +219,10 @@
     set to NULL, a memory exception is set, and -1 is returned.
     """
     # XXX always create a new string so far
-    py_str = rffi.cast(PyStringObject, ref[0])
-    if not py_str.c_buffer:
+    if pyobj_has_w_obj(ref[0]):
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
+    py_str = rffi.cast(PyStringObject, ref[0])
     try:
         py_newstr = new_empty_str(space, newsize)
     except MemoryError:
@@ -224,7 +234,7 @@
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newstr.c_buffer[i] = py_str.c_buffer[i]
+        py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newstr)
     return 0
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.1-alpha0"
-#define PYPY_VERSION_NUM  0x05030100
+#define PYPY_VERSION "5.3.2-alpha0"
+#define PYPY_VERSION_NUM  0x05030200
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -10,7 +10,6 @@
 #include 
 
 #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op))
-#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op))
 /*
 Type PyStringObject represents a character string.  An extra zero byte is
 reserved at the end to ensure it is zero-terminated, but a size is
@@ -41,12 +40,11 @@
     PyObject_VAR_HEAD
     long ob_shash;
     int ob_sstate;
-    char * buffer; /* change the name from cpython so all non-api c access is thwarted */
+    char ob_sval[1]; 
 
     /* Invariants 
-     * (not relevant in PyPy, all stringobjects are backed by a pypy object)
-     *     buffer contains space for 'ob_size+1' elements.
-     *     buffer[ob_size] == 0.
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
      *     ob_shash is the hash of the string or -1 if not computed yet.
      *     ob_sstate != 0 iff the string object is in stringobject.c's
      *       'interned' dictionary; in this case the two references
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    INTERPLEVEL_API)
+    INTERPLEVEL_API, PyVarObject)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.objectobject import W_ObjectObject
@@ -47,13 +47,16 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount and w_type is not space.w_str:
+        if pytype.c_tp_itemsize:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
+        if pytype.c_tp_itemsize:
+            pyvarobj = rffi.cast(PyVarObject, pyobj)
+            pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
         #pyobj.c_ob_pypy_link should get assigned very quickly
         pyobj.c_ob_type = pytype
@@ -152,13 +155,18 @@
 class InvalidPointerException(Exception):
     pass
 
-def create_ref(space, w_obj, itemcount=0):
+def create_ref(space, w_obj):
     """
     Allocates a PyObject, and fills its fields with info from the given
     interpreter object.
     """
     w_type = space.type(w_obj)
+    pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
+    if pytype.c_tp_itemsize != 0:
+        itemcount = space.len_w(w_obj) # PyStringObject and subclasses
+    else:
+        itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)
     track_reference(space, py_obj, w_obj)
     #
diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c
--- a/pypy/module/cpyext/src/stringobject.c
+++ b/pypy/module/cpyext/src/stringobject.c
@@ -107,7 +107,7 @@
     if (!string)
         return NULL;
 
-    s = PyString_AsString(string);
+    s = PyString_AS_STRING(string);
 
     for (f = format; *f; f++) {
         if (*f == '%') {
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+
 class AppTestStringObject(AppTestCpythonExtensionBase):
     def test_basic(self):
         module = self.import_extension('foo', [
@@ -113,6 +114,7 @@
         assert s == 'test'
 
     def test_manipulations(self):
+        import sys
         module = self.import_extension('foo', [
             ("bytearray_from_string", "METH_VARARGS",
              '''
@@ -155,7 +157,9 @@
              """)])
         assert module.bytearray_from_string("huheduwe") == "huhe"
         assert module.str_from_bytearray(bytearray('abc')) == 'abc'
-        raises(ValueError, module.str_from_bytearray, 4.0)
+        if '__pypy__' in sys.builtin_module_names:
+            # CPython only makes an assert.
+            raises(ValueError, module.str_from_bytearray, 4.0)
         ret = module.concat('abc', 'def')
         assert ret == 'abcdef'
         assert not isinstance(ret, str)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -25,14 +25,13 @@
             ("test_Size", "METH_NOARGS",
              """
                  PyObject* s = PyString_FromString("Hello world");
-                 int result = 0;
+                 int result;
                  size_t expected_size;
 
-                 if(PyString_Size(s) == 11) {
-                     result = 1;
-                 }
+                 result = PyString_Size(s);
+                
                  #ifdef PYPY_VERSION
-                    expected_size = sizeof(void*)*7;
+                    expected_size = 48;
                  #elif defined Py_DEBUG
                     expected_size = 53;
                  #else
@@ -44,7 +43,7 @@
                      result = 0;
                  }
                  Py_DECREF(s);
-                 return PyBool_FromLong(result);
+                 return PyLong_FromLong(result);
              """),
             ("test_Size_exception", "METH_NOARGS",
              """
@@ -60,7 +59,7 @@
              """)], prologue='#include ')
         assert module.get_hello1() == 'Hello world'
         assert module.get_hello2() == 'Hello world'
-        assert module.test_Size()
+        assert module.test_Size() == 11
         raises(TypeError, module.test_Size_exception)
 
         assert module.test_is_string("")
@@ -80,7 +79,7 @@
                  if (t == NULL)
                     return NULL;
                  Py_DECREF(t);
-                 c = PyString_AsString(s);
+                 c = PyString_AS_STRING(s);
                  c[0] = 'a';
                  c[1] = 'b';
                  c[2] = 0;
@@ -108,14 +107,23 @@
                 obj = (PyStringObject*)type->tp_alloc(type, 10);
                 if (PyString_GET_SIZE(obj) != 10)
                     return PyLong_FromLong(PyString_GET_SIZE(obj));
-                /* cannot work, there is only RO access
-                memcpy(PyString_AS_STRING(obj), "works", 6); */
+                /* cannot work, there is only RO access */
+                memcpy(PyString_AS_STRING(obj), "works", 6);
                 Py_INCREF(obj);
                 return (PyObject*)obj;
              """),
+            ('alloc_rw', "METH_NOARGS",
+             '''
+                PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
+                char * buf = PyString_AS_STRING(obj);
+                memcpy(PyString_AS_STRING(obj), "works", 6);
+                return (PyObject*)obj;
+             '''),
             ])
+        s = module.alloc_rw()
+        assert s == 'works' + '\x00' * 5
         s = module.tpalloc()
-        assert s == '\x00' * 10
+        assert s == 'works' + '\x00' * 5
 
     def test_AsString(self):
         module = self.import_extension('foo', [
@@ -330,27 +338,127 @@
         # doesn't really test, but if printf is enabled will prove sstate
         assert module.test_sstate()
 
+    def test_subclass(self):
+        # taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        module = self.import_extension('bar', [
+            ("newsubstr", "METH_O",
+             """
+                PyObject * obj;
+                char * data;
+                int len;
+                PyType_Ready(&PyStringArrType_Type);
+                
+                data = PyString_AS_STRING(args);
+                len = PyString_GET_SIZE(args);
+                if (data == NULL || len < 1)
+                    Py_RETURN_NONE;
+                obj = PyArray_Scalar(data, len);
+                return obj;
+             """),
+            ], prologue="""
+                #include 
+                PyTypeObject PyStringArrType_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "bar.string_",                /* tp_name*/
+                    sizeof(PyStringObject), /* tp_basicsize*/
+                    0                             /* tp_itemsize */
+                    };
+
+                    static PyObject *
+                    stringtype_repr(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    static PyObject *
+                    stringtype_str(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+                        PyObject *ret;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    PyObject *
+                    PyArray_Scalar(char *data, int n)
+                    {
+                        PyTypeObject *type = &PyStringArrType_Type;
+                        PyObject *obj;
+                        void *destptr;
+                        int type_num;
+                        int itemsize = n;
+                        obj = type->tp_alloc(type, itemsize);
+                        if (obj == NULL) {
+                            return NULL;
+                        }
+                        destptr = PyString_AS_STRING(obj);
+                        ((PyStringObject *)obj)->ob_shash = -1;
+                        memcpy(destptr, data, itemsize);
+                        return obj;
+                    }
+            """, more_init = '''
+                PyStringArrType_Type.tp_alloc = NULL;
+                PyStringArrType_Type.tp_free = NULL;
+
+                PyStringArrType_Type.tp_repr = stringtype_repr;
+                PyStringArrType_Type.tp_str = stringtype_str;
+                PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
+                PyStringArrType_Type.tp_itemsize = sizeof(char);
+                PyStringArrType_Type.tp_base = &PyString_Type;
+            ''')
+
+        a = module.newsubstr('abc')
+        assert type(a).__name__ == 'string_'
+        assert a == 'abc'
 
 class TestString(BaseApiTest):
     def test_string_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_buffer[0] = 'a'
-        py_str.c_buffer[1] = 'b'
-        py_str.c_buffer[2] = 'c'
+        py_str.c_ob_sval[0] = 'a'
+        py_str.c_ob_sval[1] = 'b'
+        py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 3)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[3] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 10)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[10] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -142,7 +142,7 @@
                     2000, 6, 6, 6, 6, 6, 6, Py_None,
                     PyDateTimeAPI->DateTimeType);
              """),
-        ])
+        ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.new_date() == datetime.date(2000, 6, 6)
         assert module.new_time() == datetime.time(6, 6, 6, 6)
@@ -241,6 +241,9 @@
                  PyObject* obj = PyDelta_FromDSU(6, 6, 6);
                  PyDateTime_Delta* delta = (PyDateTime_Delta*)obj;
 
+#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000
+                 // These macros are only defined in CPython 3.x and PyPy.
+                 // See: http://bugs.python.org/issue13727
                  PyDateTime_DELTA_GET_DAYS(obj);
                  PyDateTime_DELTA_GET_DAYS(delta);
 
@@ -249,10 +252,10 @@
 
                  PyDateTime_DELTA_GET_MICROSECONDS(obj);
                  PyDateTime_DELTA_GET_MICROSECONDS(delta);
-
+#endif
                  return obj;
              """),
-            ])
+            ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.test_date_macros() == datetime.date(2000, 6, 6)
         assert module.test_datetime_macros() == datetime.datetime(
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -5,8 +5,6 @@
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
-import sys
-
 class AppTestTypeObject(AppTestCpythonExtensionBase):
     def test_typeobject(self):
         import sys
@@ -896,7 +894,7 @@
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
-        # on cpython, the size changes (6 bytes added)
+        import sys
         module = self.import_module(name='foo')
         size = module.size_of_instances(module.fooType)
         class f1(object):
@@ -906,7 +904,11 @@
         class bar(f1, f2):
             pass
         assert bar.__base__ is f2
-        assert module.size_of_instances(bar) == size
+        # On cpython, the size changes.
+        if '__pypy__' in sys.builtin_module_names:
+            assert module.size_of_instances(bar) == size
+        else:
+            assert module.size_of_instances(bar) >= size
 
     def test_app_cant_subclass_two_types(self):
         module = self.import_module(name='foo')
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -657,6 +657,8 @@
         pto.c_tp_dealloc = llhelper(
             subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
+    if space.is_w(w_type, space.w_str):
+        pto.c_tp_itemsize = 1
     # buffer protocol
     setup_buffer_procs(space, w_type, pto)
 
@@ -695,6 +697,8 @@
     if pto.c_tp_base:
         if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize:
             pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize
+        if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
+            pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
     # will be filled later on with the correct value
     # may not be 0
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -77,7 +77,9 @@
     """
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
     s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length)
-    w_obj = space.wrap(s)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
+    w_obj.__init__(s)
     py_uni.c_hash = space.hash_w(w_obj)
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -163,14 +163,10 @@
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
             guard_true(i32, descr=...)
-            i34 = getarrayitem_raw_i(#, #, descr=)  # XXX what are these?
-            guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i35 = getarrayitem_raw_i(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, 8)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, #, descr=...)                  # XXX
             jump(..., descr=...)
         """)
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rdynload
 import sys
 
 _WIN = sys.platform == 'win32'
@@ -19,6 +20,7 @@
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
         self.debug = True
+        self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
         '__name__'              : '(space.wrap("sys"))',
@@ -85,7 +87,9 @@
 
         'float_info'            : 'system.get_float_info(space)',
         'long_info'             : 'system.get_long_info(space)',
-        'float_repr_style'      : 'system.get_float_repr_style(space)'
+        'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'getdlopenflags'        : 'system.getdlopenflags',
+        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -58,3 +58,9 @@
 
 def get_float_repr_style(space):
     return space.wrap("short")
+
+def getdlopenflags(space):
+    return space.wrap(space.sys.dlopenflags)
+
+def setdlopenflags(space, w_flags):
+    space.sys.dlopenflags = space.int_w(w_flags)
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -445,14 +445,12 @@
 
     def test_dlopenflags(self):
         import sys
-        if hasattr(sys, "setdlopenflags"):
-            assert hasattr(sys, "getdlopenflags")
-            raises(TypeError, sys.getdlopenflags, 42)
-            oldflags = sys.getdlopenflags()
-            raises(TypeError, sys.setdlopenflags)
-            sys.setdlopenflags(oldflags+1)
-            assert sys.getdlopenflags() == oldflags+1
-            sys.setdlopenflags(oldflags)
+        raises(TypeError, sys.getdlopenflags, 42)
+        oldflags = sys.getdlopenflags()
+        raises(TypeError, sys.setdlopenflags)
+        sys.setdlopenflags(oldflags+1)
+        assert sys.getdlopenflags() == oldflags+1
+        sys.setdlopenflags(oldflags)
 
     def test_refcount(self):
         import sys
@@ -610,7 +608,7 @@
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
-        
+
         class Tracer:
             def __init__(self):
                 self.events = []
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -417,11 +417,11 @@
 class FakeModule(W_Root):
     def __init__(self):
         self.w_dict = w_some_obj()
-
     def get(self, name):
         name + "xx"   # check that it's a string
         return w_some_obj()
 FakeObjSpace.sys = FakeModule()
 FakeObjSpace.sys.filesystemencoding = 'foobar'
 FakeObjSpace.sys.defaultencoding = 'ascii'
+FakeObjSpace.sys.dlopenflags = 123
 FakeObjSpace.builtin = FakeModule()
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.unicodeobject import (
     decode_object, unicode_from_encoded_object,
     unicode_from_string, getdefaultencoding)
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
 class W_AbstractBytesObject(W_Root):
@@ -30,12 +31,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.str_w(self) is space.str_w(w_other)
+        s1 = space.str_w(self)
+        s2 = space.str_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.str_w(self)))
+        s = space.str_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ord(s[0])     # base values 0-255
+            else:
+                base = 256           # empty string: base value 256
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def unicode_w(self, space):
         # Use the default encoding.
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -6,6 +6,7 @@
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 from rpython.rlib.objectmodel import r_dict
 from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash
@@ -575,6 +576,23 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_FrozensetObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty frozensets are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty frozenset: base value 259
+        uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def _newobj(self, space, w_iterable):
         """Make a new frozenset by taking ownership of 'w_iterable'."""
         if type(self) is W_FrozensetObject:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -162,7 +162,8 @@
             buffer = _get_buffer(space, w_sub)
             res = count(value, buffer, start, end)
 
-        return space.wrap(max(res, 0))
+        assert res >= 0
+        return space.wrap(res)
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -211,6 +211,7 @@
 
         check(bytearray('abc').replace('b', bytearray('d')), 'adc')
         check(bytearray('abc').replace('b', 'd'), 'adc')
+        check(bytearray('').replace('a', 'ab'), '')
 
         check(bytearray('abc').upper(), 'ABC')
         check(bytearray('ABC').lower(), 'abc')
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -186,17 +186,36 @@
     def test_id_on_strs(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        u = u"a"
-        assert id(self.unwrap_wrap_unicode(u)) == id(u)
-        s = "a"
-        assert id(self.unwrap_wrap_str(s)) == id(s)
+        for u in [u"", u"a", u"aa"]:
+            assert id(self.unwrap_wrap_unicode(u)) == id(u)
+            s = str(u)
+            assert id(self.unwrap_wrap_str(s)) == id(s)
+        #
+        assert id('') == (256 << 4) | 11     # always
+        assert id(u'') == (257 << 4) | 11
+        assert id('a') == (ord('a') << 4) | 11
+        assert id(u'\u1234') == ((~0x1234) << 4) | 11
+
+    def test_id_of_tuples(self):
+        l = []
+        x = (l,)
+        assert id(x) != id((l,))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(()) == (258 << 4) | 11     # always
+
+    def test_id_of_frozensets(self):
+        x = frozenset([4])
+        assert id(x) != id(frozenset([4]))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(frozenset()) == (259 << 4) | 11     # always
+        assert id(frozenset([])) == (259 << 4) | 11   # always
 
     def test_identity_vs_id_primitives(self):
-        if self.cpython_apptest:
-            skip("cpython behaves differently")
         import sys
-        l = range(-10, 10)
-        for i in range(10):
+        l = range(-10, 10, 2)
+        for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
             l.append(long(i))
@@ -206,18 +225,15 @@
             l.append(i - 1j)
             l.append(1 + i * 1j)
             l.append(1 - i * 1j)
-            s = str(i)
-            l.append(s)
-            u = unicode(s)
-            l.append(u)
+            l.append((i,))
+            l.append(frozenset([i]))
         l.append(-0.0)
         l.append(None)
         l.append(True)
         l.append(False)
-        s = "s"
-        l.append(s)
-        s = u"s"
-        l.append(s)
+        l.append(())
+        l.append(tuple([]))
+        l.append(frozenset())
 
         for i, a in enumerate(l):
             for b in l[i:]:
@@ -228,21 +244,18 @@
     def test_identity_vs_id_str(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        import sys
-        l = range(-10, 10)
-        for i in range(10):
-            s = str(i)
+        l = []
+        def add(s, u):
             l.append(s)
             l.append(self.unwrap_wrap_str(s))
-            u = unicode(s)
+            l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
-        s = "s"
-        l.append(s)
-        l.append(self.unwrap_wrap_str(s))
-        s = u"s"
-        l.append(s)
-        l.append(self.unwrap_wrap_unicode(s))
+            l.append(u[:1] + u[1:])
+        for i in range(3, 18):
+            add(str(i), unicode(i))
+        add("s", u"s")
+        add("", u"")
 
         for i, a in enumerate(l):
             for b in l[i:]:
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -9,7 +9,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop,
     normalize_simple_slice)
-from pypy.objspace.std.util import negate
+from pypy.objspace.std.util import negate, IDTAG_SPECIAL, IDTAG_SHIFT
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import intmask
@@ -38,6 +38,23 @@
 class W_AbstractTupleObject(W_Root):
     __slots__ = ()
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractTupleObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty tuples are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty tuple: base value 258
+        uid = (258 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def __repr__(self):
         """representation for debugging purposes"""
         reprlist = [repr(w_item) for w_item in self.tolist()]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -18,6 +18,7 @@
 from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
@@ -52,12 +53,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.unicode_w(self) is space.unicode_w(w_other)
+        s1 = space.unicode_w(self)
+        s2 = space.unicode_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.unicode_w(self)))
+        s = space.unicode_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ~ord(s[0])      # negative base values
+            else:
+                base = 257       # empty unicode string: base value 257
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def str_w(self, space):
         return space.str_w(space.str(self))
diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py
--- a/pypy/objspace/std/util.py
+++ b/pypy/objspace/std/util.py
@@ -9,6 +9,12 @@
 IDTAG_FLOAT   = 5
 IDTAG_COMPLEX = 7
 IDTAG_METHOD  = 9
+IDTAG_SPECIAL = 11    # -1 - (-maxunicode-1): unichar
+                      # 0 - 255: char
+                      # 256: empty string
+                      # 257: empty unicode
+                      # 258: empty tuple
+                      # 259: empty frozenset
 
 CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
 BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,9 +1,9 @@
 # Edit these appropriately before running this script
 maj=5
 min=3
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState,\
+     VArrayStateInfo, not_virtual, VirtualState,\
      GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
@@ -31,10 +31,10 @@
     def setup_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info = not_virtual(self.cpu, 'r', value)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info2 = not_virtual(self.cpu, 'r', value)
     
     def guards(self, info1, info2, box, runtime_box, expected, inputargs=None):
         if inputargs is None:
@@ -80,7 +80,7 @@
     def test_make_inputargs(self):
         optimizer = FakeOptimizer(self.cpu)
         args = [InputArgInt()]
-        info0 = NotVirtualStateInfo(self.cpu, args[0].type, None)
+        info0 = not_virtual(self.cpu, args[0].type, None)
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
@@ -108,8 +108,8 @@
             assert info1 in state.bad and info2 in state.bad
 
         for BoxType in (InputArgInt, InputArgFloat, InputArgRef):
-            info1 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
-            info2 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
+            info1 = not_virtual(self.cpu, BoxType.type, None)
+            info2 = not_virtual(self.cpu, BoxType.type, None)
             postest(info1, info2)
 
         info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42)
@@ -126,9 +126,9 @@
 
     def test_NotVirtualStateInfo_generalization(self):
         def isgeneral(tp1, info1, tp2, info2):
-            info1 = NotVirtualStateInfo(self.cpu, tp1, info1)
+            info1 = not_virtual(self.cpu, tp1, info1)
             info1.position = 0
-            info2 = NotVirtualStateInfo(self.cpu, tp2, info2)
+            info2 = not_virtual(self.cpu, tp2, info2)
             info2.position = 0
             return VirtualState([info1]).generalization_of(VirtualState([info2]), FakeOptimizer(self.cpu))
 
@@ -166,8 +166,8 @@
         assert not isgeneral('r', value1, 'r', value2)
 
     def test_field_matching_generalization(self):
-        const1 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(1))
-        const2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(2))
+        const1 = not_virtual(self.cpu, 'i', ConstIntBound(1))
+        const2 = not_virtual(self.cpu, 'i', ConstIntBound(2))
         const1.position = const2.position = 1
         self.check_invalid(const1, const2)
         self.check_invalid(const2, const1)
@@ -192,16 +192,16 @@
 
     def test_known_class_generalization(self):
         knownclass1 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info1 = NotVirtualStateInfo(self.cpu, 'r', knownclass1)
+        info1 = not_virtual(self.cpu, 'r', knownclass1)
         info1.position = 0
         knownclass2 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info2 = NotVirtualStateInfo(self.cpu, 'r', knownclass2)
+        info2 = not_virtual(self.cpu, 'r', knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
         knownclass3 = info.InstancePtrInfo(None, ConstPtr(self.myptr2))
-        info3 = NotVirtualStateInfo(self.cpu, 'r', knownclass3)
+        info3 = not_virtual(self.cpu, 'r', knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
         self.check_invalid(info2, info3)
@@ -222,26 +222,26 @@
         #unknown_val = PtrOptValue(self.nodebox)
         #unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
         opt = FakeOptimizer(self.cpu)
-        unknown_info = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info = not_virtual(self.cpu, 'r', None)
 
-        nonnull_info = NotVirtualStateInfo(self.cpu, 'r', info.NonNullPtrInfo())
+        nonnull_info = not_virtual(self.cpu, 'r', info.NonNullPtrInfo())
 
         classbox1 = self.cpu.ts.cls_of_box(ConstPtr(self.nodeaddr))
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox1))
+        knownclass_info = not_virtual(self.cpu, 'r',
+                                      info.InstancePtrInfo(None, classbox1))
         classbox2 = self.cpu.ts.cls_of_box(ConstPtr(self.node2addr))
-        knownclass2_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox2))
+        knownclass2_info = not_virtual(self.cpu, 'r',
+                                       info.InstancePtrInfo(None, classbox2))
 
-        constant_info = NotVirtualStateInfo(self.cpu, 'i',
-                                            ConstIntBound(1))
-        constant_ptr_info = NotVirtualStateInfo(self.cpu, 'r',
+        constant_info = not_virtual(self.cpu, 'i',
+                                    ConstIntBound(1))
+        constant_ptr_info = not_virtual(self.cpu, 'r',
                                     info.ConstPtrInfo(ConstPtr(self.nodeaddr)))
         constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr))
-        constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val)
-        constclass2_info = NotVirtualStateInfo(self.cpu, 'r',
+        constclass_info = not_virtual(self.cpu, 'r', constclass_val)
+        constclass2_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.node2addr)))
-        constantnull_info = NotVirtualStateInfo(self.cpu, 'r',
+        constantnull_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.nullptr)))
 
         # unknown unknown
@@ -260,9 +260,10 @@
         self.check_no_guards(unknown_info, knownclass_info)
 
         # unknown constant
-        self.check_no_guards(unknown_info, constant_info,
+        unknown_info_int = not_virtual(self.cpu, 'i', None)
+        self.check_no_guards(unknown_info_int, constant_info,
                              ConstInt(1), ConstIntBound(1))
-        self.check_no_guards(unknown_info, constant_info)
+        self.check_no_guards(unknown_info_int, constant_info)
 
 
         # nonnull unknown
@@ -293,11 +294,11 @@
         const_nonnull = ConstPtr(self.nodeaddr)
         const_nonnull2 = ConstPtr(self.node2addr)
         const_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
-        self.check_no_guards(nonnull_info, constant_info, const_nonnull,
+        self.check_no_guards(nonnull_info, constant_ptr_info, const_nonnull,
                              info.ConstPtrInfo(const_nonnull))
         self.check_invalid(nonnull_info, constantnull_info, const_null,
                            info.ConstPtrInfo(const_null))
-        self.check_no_guards(nonnull_info, constant_info)
+        self.check_no_guards(nonnull_info, constant_ptr_info)
         self.check_invalid(nonnull_info, constantnull_info)
 
 
@@ -392,8 +393,8 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', IntUnbounded())
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', IntUnbounded())
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -408,18 +409,18 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(10000))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(10000))
         self.check_invalid(info1, info2)
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(11))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(11))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value1 = info.InstancePtrInfo(None, classbox)
-        info1 = NotVirtualStateInfo(self.cpu, 'r', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        info1 = not_virtual(self.cpu, 'r', value1)
+        info2 = not_virtual(self.cpu, 'r', None)
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []        
@@ -456,18 +457,18 @@
     def test_equal_inputargs(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
+        unknown_info2 = not_virtual(self.cpu, 'r', None)
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate3.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -494,9 +495,9 @@
     def test_generate_guards_on_virtual_fields_matches_array(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         descr = ArrayDescr(lltype.GcArray(llmemory.GCREF), self.cpu)
@@ -524,9 +525,9 @@
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
@@ -552,9 +553,9 @@
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         structdescr = self.nodesize
@@ -583,9 +584,9 @@
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         NODE = lltype.Struct('NODE', ('x', llmemory.GCREF))
@@ -627,7 +628,7 @@
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info2.fieldstate = [unknown_info1, unknown_info1]
         vstate2 = VirtualState([info2])
@@ -636,9 +637,9 @@
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info3 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info2 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info3.fieldstate = [unknown_info1, unknown_info2]
         vstate3 = VirtualState([info3])        
@@ -651,7 +652,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -659,7 +660,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -671,7 +672,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -679,7 +680,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -691,7 +692,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -699,24 +700,24 @@
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
         assert not vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
-        
+
     def test_nonvirtual_is_not_virtual(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        info2 = not_virtual(self.cpu, 'r', value)
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
@@ -727,7 +728,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -735,7 +736,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -747,7 +748,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -755,7 +756,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -793,7 +794,7 @@
     def test_crash_varay_clear(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
         innerinfo1.position_in_notvirtuals = 0
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -350,40 +350,23 @@
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
 
+
+def not_virtual(cpu, type, info):
+    if type == 'i':
+        return NotVirtualStateInfoInt(cpu, type, info)
+    if type == 'r':
+        return NotVirtualStateInfoPtr(cpu, type, info)
+    return NotVirtualStateInfo(cpu, type, info)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
-    lenbound = None
-    intbound = None
     level = LEVEL_UNKNOWN
     constbox = None
-    known_class = None
-    
+
     def __init__(self, cpu, type, info):
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
-            if type == 'r':
-                self.known_class = info.get_known_class(cpu)
-            elif type == 'i':
-                self.intbound = info
-        elif type == 'r':
-            if info:
-                self.known_class = info.get_known_class(cpu)
-                if self.known_class:
-                    self.level = LEVEL_KNOWNCLASS
-                elif info.is_nonnull():
-                    self.level = LEVEL_NONNULL
-                self.lenbound = info.getlenbound(None)
-        elif type == 'i':
-            if isinstance(info, IntBound):
-                if info.lower < MININT / 2:
-                    info.lower = MININT
-                if info.upper > MAXINT / 2:
-                    info.upper = MAXINT
-                self.intbound = info
-        elif type == 'f':
-            if info and info.is_constant():
-                self.level = LEVEL_CONSTANT
-                self.constbox = info.getconst()
 
     def is_const(self):
         return self.constbox is not None
@@ -394,84 +377,15 @@
     def _generate_guards(self, other, box, runtime_box, state):
         # XXX This will always retrace instead of forcing anything which
         # might be what we want sometimes?
-        if not isinstance(other, NotVirtualStateInfo):
-            raise VirtualStatesCantMatch(
-                    'The VirtualStates does not match as a ' +
-                    'virtual appears where a pointer is needed ' +
-                    'and it is too late to force it.')
-
-
         extra_guards = state.extra_guards
-        cpu = state.cpu
-        if self.lenbound:
-            if other.lenbound is None:
-                other_bound = IntLowerBound(0)
-            else:
-                other_bound = other.lenbound
-            if not self.lenbound.contains_bound(other_bound):
-                raise VirtualStatesCantMatch("length bound does not match")
-
         if self.level == LEVEL_UNKNOWN:
-            # confusingly enough, this is done also for pointers
-            # which have the full range as the "bound", so it always works
-            return self._generate_guards_intbounds(other, box, runtime_box,
-                                                   extra_guards,
-                                                   state.optimizer)
-
-        # the following conditions often peek into the runtime value that the
-        # box had when tracing. This value is only used as an educated guess.
-        # It is used here to choose between either emitting a guard and jumping
-        # to an existing compiled loop or retracing the loop. Both alternatives
-        # will always generate correct behaviour, but performance will differ.
-        elif self.level == LEVEL_NONNULL:
-            if other.level == LEVEL_UNKNOWN:
-                if runtime_box is not None and runtime_box.nonnull():
-                    op = ResOperation(rop.GUARD_NONNULL, [box])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other not known to be nonnull")
-            elif other.level == LEVEL_NONNULL:
-                return
-            elif other.level == LEVEL_KNOWNCLASS:
-                return # implies nonnull
-            else:
-                assert other.level == LEVEL_CONSTANT
-                assert other.constbox
-                if not other.constbox.nonnull():
-                    raise VirtualStatesCantMatch("constant is null")
-                return
-
-        elif self.level == LEVEL_KNOWNCLASS:
-            if other.level == LEVEL_UNKNOWN:
-                if (runtime_box and runtime_box.nonnull() and
-              self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_NONNULL:
-                if (runtime_box and self.known_class.same_constant(
-                        cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_KNOWNCLASS:
-                if self.known_class.same_constant(other.known_class):
-                    return
-                raise VirtualStatesCantMatch("classes don't match")
-            else:
-                assert other.level == LEVEL_CONSTANT
-                if (other.constbox.nonnull() and
-                        self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))):
-                    return
-                else:
-                    raise VirtualStatesCantMatch("classes don't match")
-
+            return self._generate_guards_unkown(other, box, runtime_box,
+                                                extra_guards,
+                                                state)
         else:
+            if not isinstance(other, NotVirtualStateInfo):
+                raise VirtualStatesCantMatch(
+                        'comparing a constant against something that is a virtual')
             assert self.level == LEVEL_CONSTANT
             if other.level == LEVEL_CONSTANT:
                 if self.constbox.same_constant(other.constbox):
@@ -485,19 +399,9 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, box, runtime_box, extra_guards,
-                                   optimizer):
-        if self.intbound is None:
-            return
-        if self.intbound.contains_bound(other.intbound):
-            return
-        if (runtime_box is not None and
-            self.intbound.contains(runtime_box.getint())):
-            # this may generate a few more guards than needed, but they are
-            # optimized away when emitting them
-            self.intbound.make_guards(box, extra_guards, optimizer)
-            return
-        raise VirtualStatesCantMatch("intbounds don't match")
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                state):
+        return
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         if self.level == LEVEL_CONSTANT:
@@ -553,8 +457,145 @@
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
 
-        debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
-                    ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
+        result = indent + mark + 'NotVirtualStateInfo(%d' % self.position + ', ' + l
+        extra = self._extra_repr()
+        if extra:
+            result += ', ' + extra
+        result += lb + ')'
+        debug_print(result)
+
+class NotVirtualStateInfoInt(NotVirtualStateInfo):
+    intbound = None
+
+    def __init__(self, cpu, type, info):
+        NotVirtualStateInfo.__init__(self, cpu, type, info)
+        assert type == 'i'
+        if isinstance(info, IntBound):
+            if info.lower < MININT / 2:
+                info.lower = MININT
+            if info.upper > MAXINT / 2:
+                info.upper = MAXINT
+            self.intbound = info
+
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                state):
+        other_intbound = None
+        if isinstance(other, NotVirtualStateInfoInt):
+            other_intbound = other.intbound
+        if self.intbound is None:
+            return
+        if self.intbound.contains_bound(other_intbound):
+            return
+        if (runtime_box is not None and
+            self.intbound.contains(runtime_box.getint())):
+            # this may generate a few more guards than needed, but they are
+            # optimized away when emitting them
+            self.intbound.make_guards(box, extra_guards, state.optimizer)
+            return
+        raise VirtualStatesCantMatch("intbounds don't match")
+
+    def _extra_repr(self):
+        return self.intbound.__repr__()
+
+
+class NotVirtualStateInfoPtr(NotVirtualStateInfo):
+    lenbound = None
+    known_class = None
+
+    def __init__(self, cpu, type, info):
+        if info:
+            self.known_class = info.get_known_class(cpu)
+            if self.known_class:
+                self.level = LEVEL_KNOWNCLASS
+            elif info.is_nonnull():
+                self.level = LEVEL_NONNULL
+            self.lenbound = info.getlenbound(None)
+        # might set it to LEVEL_CONSTANT
+        NotVirtualStateInfo.__init__(self, cpu, type, info)

From pypy.commits at gmail.com  Mon Jun 20 23:35:07 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 20:35:07 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup: fix warnings
Message-ID: <5768b5eb.4aa71c0a.3bd5d.34ea@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup
Changeset: r85300:0f340d606731
Date: 2016-06-21 04:34 +0100
http://bitbucket.org/pypy/pypy/changeset/0f340d606731/

Log:	fix warnings

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -29,7 +29,7 @@
                  size_t expected_size;
 
                  result = PyString_Size(s);
-                
+
                  #ifdef PYPY_VERSION
                     expected_size = 48;
                  #elif defined Py_DEBUG
@@ -115,7 +115,6 @@
             ('alloc_rw', "METH_NOARGS",
              '''
                 PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
-                char * buf = PyString_AS_STRING(obj);
                 memcpy(PyString_AS_STRING(obj), "works", 6);
                 return (PyObject*)obj;
              '''),
@@ -347,7 +346,7 @@
                 char * data;
                 int len;
                 PyType_Ready(&PyStringArrType_Type);
-                
+
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
                 if (data == NULL || len < 1)
@@ -371,7 +370,6 @@
                         const char *dptr, *ip;
                         int len;
                         PyObject *new;
-                        PyObject *ret;
 
                         ip = dptr = PyString_AS_STRING(self);
                         len = PyString_GET_SIZE(self);
@@ -392,7 +390,6 @@
                         const char *dptr, *ip;
                         int len;
                         PyObject *new;
-                        PyObject *ret;
 
                         ip = dptr = PyString_AS_STRING(self);
                         len = PyString_GET_SIZE(self);
@@ -413,7 +410,6 @@
                         PyTypeObject *type = &PyStringArrType_Type;
                         PyObject *obj;
                         void *destptr;
-                        int type_num;
                         int itemsize = n;
                         obj = type->tp_alloc(type, itemsize);
                         if (obj == NULL) {

From pypy.commits at gmail.com  Mon Jun 20 23:37:20 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Jun 2016 20:37:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge branch testing-cleanup
Message-ID: <5768b670.46c21c0a.ad251.35f3@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r85301:2992076e0434
Date: 2016-06-21 04:36 +0100
http://bitbucket.org/pypy/pypy/changeset/2992076e0434/

Log:	Merge branch testing-cleanup

	Don't use interp-level RPython machinery to test building app-level
	extensions.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -800,6 +800,21 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
+def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+    from rpython.rlib import rgil
+    # see "Handling of the GIL" above
+    assert cpyext_glob_tid_ptr[0] == 0
+    if pygilstate_release:
+        from pypy.module.cpyext import pystate
+        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
+    else:
+        unlock = gil_release or _gil_auto
+    if unlock:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
+
+
 def make_wrapper_second_level(space, argtypesw, restype,
                               result_kind, error_value, gil):
     from rpython.rlib import rgil
@@ -827,6 +842,7 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
+        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -839,7 +855,6 @@
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
-            from pypy.module.cpyext import pystate
             if cpyext_glob_tid_ptr[0] == tid:
                 cpyext_glob_tid_ptr[0] = 0
                 args += (pystate.PyGILState_LOCKED,)
@@ -850,6 +865,10 @@
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
+        if pygilstate_release:
+            gilstate = rffi.cast(lltype.Signed, args[-1])
+        else:
+            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -919,24 +938,13 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
+            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        # see "Handling of the GIL" above
-        assert cpyext_glob_tid_ptr[0] == 0
-        if pygilstate_release:
-            from pypy.module.cpyext import pystate
-            arg = rffi.cast(lltype.Signed, args[-1])
-            unlock = (arg == pystate.PyGILState_UNLOCKED)
-        else:
-            unlock = gil_release or _gil_auto
-        if unlock:
-            rgil.release()
-        else:
-            cpyext_glob_tid_ptr[0] = tid
-
+        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -207,6 +207,7 @@
 PyGILState_STATE = rffi.INT
 PyGILState_LOCKED = 0
 PyGILState_UNLOCKED = 1
+PyGILState_IGNORE = 2
 
 ExecutionContext.cpyext_gilstate_counter_noleave = 0
 
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import os
 import pytest
 
 def pytest_configure(config):
@@ -21,3 +21,14 @@
 def pytest_funcarg__api(request):
     return request.cls.api
 
+if os.name == 'nt':
+    @pytest.yield_fixture(autouse=True, scope='session')
+    def prevent_dialog_box():
+        """Do not open dreaded dialog box on segfault on Windows"""
+        import ctypes
+        SEM_NOGPFAULTERRORBOX = 0x0002  # From MSDN
+        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
+        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
+        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
+        yield
+        ctypes.windll.kernel32.SetErrorMode(old_err_mode)
diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/support.py
@@ -0,0 +1,68 @@
+import os
+import py
+from sys import platform
+
+if os.name != 'nt':
+    so_ext = 'so'
+else:
+    so_ext = 'dll'
+
+def c_compile(cfilenames, outputfilename,
+        compile_extra=None, link_extra=None,
+        include_dirs=None, libraries=None, library_dirs=None):
+    compile_extra = compile_extra or []
+    link_extra = link_extra or []
+    include_dirs = include_dirs or []
+    libraries = libraries or []
+    library_dirs = library_dirs or []
+    if platform == 'win32':
+        link_extra = link_extra + ['/DEBUG'] # generate .pdb file
+    if platform == 'darwin':
+        # support Fink & Darwinports
+        for s in ('/sw/', '/opt/local/'):
+            if (s + 'include' not in include_dirs
+                    and os.path.exists(s + 'include')):
+                include_dirs.append(s + 'include')
+            if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
+                library_dirs.append(s + 'lib')
+
+    outputfilename = py.path.local(outputfilename).new(ext=so_ext)
+    saved_environ = os.environ.copy()
+    try:
+        _build(
+            cfilenames, outputfilename,
+            compile_extra, link_extra,
+            include_dirs, libraries, library_dirs)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
+    return outputfilename
+
+def _build(cfilenames, outputfilename, compile_extra, link_extra,
+        include_dirs, libraries, library_dirs):
+    from distutils.ccompiler import new_compiler
+    from distutils import sysconfig
+    compiler = new_compiler(force=1)
+    sysconfig.customize_compiler(compiler) # XXX
+    objects = []
+    for cfile in cfilenames:
+        cfile = py.path.local(cfile)
+        old = cfile.dirpath().chdir()
+        try:
+            res = compiler.compile([cfile.basename],
+                include_dirs=include_dirs, extra_preargs=compile_extra)
+            assert len(res) == 1
+            cobjfile = py.path.local(res[0])
+            assert cobjfile.check()
+            objects.append(str(cobjfile))
+        finally:
+            old.chdir()
+
+    compiler.link_shared_object(
+        objects, str(outputfilename),
+        libraries=libraries,
+        extra_preargs=link_extra,
+        library_dirs=library_dirs)
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -54,7 +54,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyByteArray_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -85,7 +84,6 @@
             ("mutable", "METH_NOARGS",
              """
                 PyObject *base;
-                char * p_str;
                 base = PyByteArray_FromStringAndSize("test", 10);
                 if (PyByteArray_GET_SIZE(base) != 10)
                     return PyLong_FromLong(-PyByteArray_GET_SIZE(base));
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -29,7 +29,7 @@
                  size_t expected_size;
 
                  result = PyString_Size(s);
-                
+
                  #ifdef PYPY_VERSION
                     expected_size = 48;
                  #elif defined Py_DEBUG
@@ -39,7 +39,7 @@
                  #endif
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
-                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); 
+                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -48,7 +48,7 @@
             ("test_Size_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyString_Size(f);
+                 PyString_Size(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -71,7 +71,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyString_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -99,7 +98,6 @@
                 PyObject *base;
                 PyTypeObject * type;
                 PyStringObject *obj;
-                char * p_str;
                 base = PyString_FromString("test");
                 if (PyString_GET_SIZE(base) != 4)
                     return PyLong_FromLong(-PyString_GET_SIZE(base));
@@ -117,7 +115,6 @@
             ('alloc_rw', "METH_NOARGS",
              '''
                 PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
-                char * buf = PyString_AS_STRING(obj);
                 memcpy(PyString_AS_STRING(obj), "works", 6);
                 return (PyObject*)obj;
              '''),
@@ -320,17 +317,17 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyStringObject*)obj)->ob_shash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ("test_sstate", "METH_NOARGS",
              '''
                 PyObject *s = PyString_FromString("xyz");
-                int sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*int sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 PyString_InternInPlace(&s);
-                sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 Py_DECREF(s);
                 return PyBool_FromLong(1);
              '''),
@@ -349,7 +346,7 @@
                 char * data;
                 int len;
                 PyType_Ready(&PyStringArrType_Type);
-                
+
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
                 if (data == NULL || len < 1)
@@ -373,7 +370,6 @@
                         const char *dptr, *ip;
                         int len;
                         PyObject *new;
-                        PyObject *ret;
 
                         ip = dptr = PyString_AS_STRING(self);
                         len = PyString_GET_SIZE(self);
@@ -394,7 +390,6 @@
                         const char *dptr, *ip;
                         int len;
                         PyObject *new;
-                        PyObject *ret;
 
                         ip = dptr = PyString_AS_STRING(self);
                         len = PyString_GET_SIZE(self);
@@ -415,7 +410,6 @@
                         PyTypeObject *type = &PyStringArrType_Type;
                         PyObject *obj;
                         void *destptr;
-                        int type_num;
                         int itemsize = n;
                         obj = type->tp_alloc(type, itemsize);
                         if (obj == NULL) {
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,8 +7,6 @@
 from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import platform
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -18,20 +16,7 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-def setup_module(module):
-    if os.name == 'nt':
-        # Do not open dreaded dialog box on segfault
-        import ctypes
-        SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
-        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
-        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
-        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
-        module.old_err_mode = old_err_mode
-
-def teardown_module(module):
-    if os.name == 'nt':
-        import ctypes
-        ctypes.windll.kernel32.SetErrorMode(module.old_err_mode)
+from .support import c_compile
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -46,7 +31,30 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
-def compile_extension_module(space, modname, include_dirs=[], **kwds):
+def convert_sources_to_files(sources, dirname):
+    files = []
+    for i, source in enumerate(sources):
+        filename = dirname / ('source_%d.c' % i)
+        with filename.open('w') as f:
+            f.write(str(source))
+        files.append(filename)
+    return files
+
+def create_so(modname, include_dirs, source_strings=None, source_files=None,
+        compile_extra=None, link_extra=None, libraries=None):
+    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+    if source_strings:
+        assert not source_files
+        files = convert_sources_to_files(source_strings, dirname)
+        source_files = files
+    soname = c_compile(source_files, outputfilename=str(dirname/modname),
+        compile_extra=compile_extra, link_extra=link_extra,
+        include_dirs=include_dirs,
+        libraries=libraries)
+    return soname
+
+def compile_extension_module(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -60,38 +68,36 @@
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        libraries = [api_library]
         # '%s' undefined; assuming extern returning int
-        kwds["compile_extra"] = ["/we4013"]
+        compile_extra = ["/we4013"]
         # prevent linking with PythonXX.lib
         w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
                               (space.int_w(w_maj), space.int_w(w_min))]
-    elif sys.platform == 'darwin':
-        kwds["link_files"] = [str(api_library + '.dylib')]
     else:
-        kwds["link_files"] = [str(api_library + '.so')]
+        libraries = []
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
-            kwds["link_extra"]=["-g"]
+            compile_extra = ["-Werror", "-g", "-O0", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs=api.include_dirs + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=api.include_dirs + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra,
+            libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
     return str(pydname)
 
-def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds):
+def compile_extension_module_applevel(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -103,24 +109,23 @@
     build the module (so specify your source with one of those).
     """
     if sys.platform == 'win32':
-        kwds["compile_extra"] = ["/we4013"]
-        kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
+        compile_extra = ["/we4013"]
+        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
     elif sys.platform == 'darwin':
+        compile_extra = link_extra = None
         pass
     elif sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"]
+        compile_extra = [
+            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
+        link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs = [space.include_dir] + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=[space.include_dir] + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra)
     return str(soname)
 
 def freeze_refcnts(self):
@@ -285,8 +290,8 @@
                 separate_module_sources = []
             pydname = self.compile_extension_module(
                 space, name,
-                separate_module_files=separate_module_files,
-                separate_module_sources=separate_module_sources)
+                source_files=separate_module_files,
+                source_strings=separate_module_sources)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -315,6 +320,11 @@
                 /* fix for cpython 2.7 Python.h if running tests with -A
                    since pypy compiles with -fvisibility-hidden */
                 #undef PyMODINIT_FUNC
+                #ifdef __GNUC__
+                #  define RPY_EXPORTED extern __attribute__((visibility("default")))
+                #else
+                #  define RPY_EXPORTED extern __declspec(dllexport)
+                #endif
                 #define PyMODINIT_FUNC RPY_EXPORTED void
 
                 %(body)s
@@ -326,16 +336,16 @@
                 """ % dict(name=name, init=init, body=body,
                            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
                                             if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(separate_module_sources=[code])
+                kwds = dict(source_strings=[code])
             else:
                 assert not PY_SSIZE_T_CLEAN
                 if filename is None:
                     filename = name
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(separate_module_files=[filename])
-            kwds['include_dirs'] = include_dirs
-            mod = self.compile_extension_module(space, name, **kwds)
+                kwds = dict(source_files=[filename])
+            mod = self.compile_extension_module(space, name,
+                    include_dirs=include_dirs, **kwds)
 
             if load_it:
                 if self.runappdirect:
@@ -975,7 +985,7 @@
             ('bar', 'METH_NOARGS',
              '''
              /* reuse a name that is #defined in structmember.h */
-             int RO;
+             int RO = 0; (void)RO;
              Py_RETURN_NONE;
              '''
              ),
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py
--- a/pypy/module/cpyext/test/test_frameobject.py
+++ b/pypy/module/cpyext/test/test_frameobject.py
@@ -13,7 +13,7 @@
                  PyObject *empty_string = PyString_FromString("");
                  PyObject *empty_tuple = PyTuple_New(0);
                  PyCodeObject *py_code;
-                 PyFrameObject *py_frame;
+                 PyFrameObject *py_frame = NULL;
 
                  py_code = PyCode_New(
                      0,            /*int argcount,*/
@@ -75,7 +75,7 @@
              """
                  int check;
                  PyObject *type, *value, *tb;
-                 PyObject *ret = PyRun_String("XXX", Py_eval_input, 
+                 PyObject *ret = PyRun_String("XXX", Py_eval_input,
                                               Py_None, Py_None);
                  if (ret) {
                      Py_DECREF(ret);
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -149,7 +149,6 @@
         pybuffer = self.import_parser(
             '''
             Py_buffer buf1, buf2, buf3;
-            PyObject *result;
             if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) {
                 return NULL;
             }
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.descriptor import get_dtype_cache
-import pypy.module.micronumpy.constants as NPY 
+import pypy.module.micronumpy.constants as NPY
 
 def scalar(space):
     dtype = get_dtype_cache(space).w_float64dtype
@@ -237,7 +237,7 @@
             except:
                 skip('numpy not importable')
         else:
-            numpy_incl = os.path.abspath(os.path.dirname(__file__) + 
+            numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
             assert os.path.exists(numpy_incl)
             cls.w_numpy_include = cls.space.wrap([numpy_incl])
@@ -273,7 +273,7 @@
                 {
                     /* Should have failed */
                     Py_DECREF(obj1);
-                    return NULL; 
+                    return NULL;
                 }
                 return obj1;
                 '''
@@ -300,14 +300,14 @@
                 ),
                 ("test_DescrFromType", "METH_O",
                 """
-                    Signed typenum = PyInt_AsLong(args);
+                    long typenum = PyInt_AsLong(args);
                     return PyArray_DescrFromType(typenum);
                 """
                 ),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -315,7 +315,7 @@
                     #define PyArray_FromObject _PyArray_FromObject
                     #define PyArray_FromAny _PyArray_FromAny
                 #endif
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -349,14 +349,14 @@
                     Py_INCREF(obj);
                     return obj;
                 '''),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -403,14 +403,14 @@
                 void *array_data[] = {NULL, NULL};
                 return PyUFunc_FromFuncAndDataAndSignature(funcs,
                                     array_data, types, 1, 1, 1, PyUFunc_None,
-                                    "float_3x3", 
-                                    "a ufunc that tests a more complicated signature", 
+                                    "float_3x3",
+                                    "a ufunc that tests a more complicated signature",
                                     0, "(m,m)->(m,m)");
                 """),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -480,7 +480,7 @@
                             res += +10;
                     *((float *)args[1]) = res;
                 };
-                            
+
                 ''',  more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -127,12 +127,12 @@
         test_compare(1, 2)
         test_compare(2, 2)
         test_compare('2', '1')
-        
+
         w_i = space.wrap(1)
         assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1
         assert api.PyErr_Occurred() is space.w_SystemError
         api.PyErr_Clear()
-        
+
     def test_IsInstance(self, space, api):
         assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1
         assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0
@@ -165,7 +165,7 @@
             return File""")
         w_f = space.call_function(w_File)
         assert api.PyObject_AsFileDescriptor(w_f) == 42
-    
+
     def test_hash(self, space, api):
         assert api.PyObject_Hash(space.wrap(72)) == 72
         assert api.PyObject_Hash(space.wrap(-1)) == -1
@@ -250,7 +250,7 @@
                  if (copy != orig)
                      PyObject_Free(copy);
                  PyObject_Free(orig);
-                 return ret;  
+                 return ret;
              """)])
         x = module.realloctest()
         assert x == 'hello world\x00'
@@ -425,7 +425,6 @@
                  """
     Py_buffer buf;
     PyObject *str = PyString_FromString("hello, world.");
-    PyObject *result;
 
     if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13,
                           1, PyBUF_WRITABLE)) {
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -130,7 +130,7 @@
         module = self.import_extension('foo', [
             ("run", "METH_NOARGS",
              """
-                long prev, next;
+                long prev;
                 PyObject *t = PyTuple_New(1);
                 prev = Py_True->ob_refcnt;
                 Py_INCREF(Py_True);
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -735,7 +735,6 @@
              """
                 IntLikeObject *intObj;
                 int intval;
-                PyObject *name;
 
                 if (!PyArg_ParseTuple(args, "i", &intval))
                     return NULL;
@@ -1060,7 +1059,6 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                PyObject *obj;
                 FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
                 FooType_Type.tp_base = &PyType_Type;
                 if (PyType_Ready(&FooType_Type) < 0) return NULL;
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -35,7 +35,7 @@
             ("test_GetSize_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyUnicode_GetSize(f);
+                 PyUnicode_GetSize(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -57,7 +57,6 @@
              """
                  PyObject *s, *t;
                  Py_UNICODE* c;
-                 Py_ssize_t len;
 
                  s = PyUnicode_FromUnicode(NULL, 4);
                  if (s == NULL)
@@ -84,7 +83,7 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyUnicodeObject*)obj)->hash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ])
@@ -234,13 +233,13 @@
         w_res = api.PyUnicode_AsUTF8String(w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\tm'
-    
+
     def test_decode_utf8(self, space, api):
         u = rffi.str2charp(u'sp\x134m'.encode("utf-8"))
         w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp\x134m'
-        
+
         w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
@@ -405,7 +404,7 @@
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
         result = api.PyUnicode_AsASCIIString(w_ustr)
-        
+
         assert space.eq_w(space.wrap(ustr), result)
 
         w_ustr = space.wrap(u"abcd\xe9f")

From pypy.commits at gmail.com  Tue Jun 21 06:58:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 03:58:41 -0700 (PDT)
Subject: [pypy-commit] pypy new-jit-log: catchup default
Message-ID: <57691de1.820b1c0a.ee8dc.ffffd1d2@mx.google.com>

Author: Richard Plangger 
Branch: new-jit-log
Changeset: r85302:bcdc23248f13
Date: 2016-06-21 12:57 +0200
http://bitbucket.org/pypy/pypy/changeset/bcdc23248f13/

Log:	catchup default

diff too long, truncating to 2000 out of 3966 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -76,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -110,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -120,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -153,28 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
 
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
-
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,10 +1,17 @@
-=========================
+==========================
 What's new in PyPy2.7 5.3+
-=========================
+==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
 .. branch: fix-gen-dfa
 
 Resolves an issue with the generator script to build the dfa for Python syntax.
@@ -19,3 +26,19 @@
 .. branch: s390x-5.3-catchup
 
 Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -458,14 +458,17 @@
         decl = str(decl) + "\n"
         yield self.st, decl, 'x', (1, 2, 3, 4)
 
+    def test_closure_error(self):
         source = """if 1:
         def f(a):
             del a
             def x():
                 a
         """
-        exc = py.test.raises(SyntaxError, self.run, source).value
-        assert exc.msg == "Can't delete variable used in nested scopes: 'a'"
+        with py.test.raises(SyntaxError) as excinfo:
+            self.run(source)
+        msg = excinfo.value.msg
+        assert msg == "Can't delete variable used in nested scopes: 'a'"
 
     def test_try_except_finally(self):
         yield self.simple_test, """
@@ -879,7 +882,20 @@
         """
         self.simple_test(source, 'ok', 1)
 
-    def test_remove_docstring(self):
+    @py.test.mark.parametrize('expr, result', [
+        ("f1.__doc__", None),
+        ("f2.__doc__", 'docstring'),
+        ("f2()", 'docstring'),
+        ("f3.__doc__", None),
+        ("f3()", 'bar'),
+        ("C1.__doc__", None),
+        ("C2.__doc__", 'docstring'),
+        ("C3.field", 'not docstring'),
+        ("C4.field", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("C4.__doc__", 'docstring'),
+        ("__doc__", None),])
+    def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():
             'docstring'
@@ -903,19 +919,7 @@
         code_w.remove_docstrings(self.space)
         dict_w = self.space.newdict();
         code_w.exec_code(self.space, dict_w, dict_w)
-
-        yield self.check, dict_w, "f1.__doc__", None
-        yield self.check, dict_w, "f2.__doc__", 'docstring'
-        yield self.check, dict_w, "f2()", 'docstring'
-        yield self.check, dict_w, "f3.__doc__", None
-        yield self.check, dict_w, "f3()", 'bar'
-        yield self.check, dict_w, "C1.__doc__", None
-        yield self.check, dict_w, "C2.__doc__", 'docstring'
-        yield self.check, dict_w, "C3.field", 'not docstring'
-        yield self.check, dict_w, "C4.field", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "C4.__doc__", 'docstring'
-        yield self.check, dict_w, "__doc__", None
+        self.check(dict_w, expr, result)
 
     def test_assert_skipping(self):
         space = self.space
@@ -1111,7 +1115,7 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
-        
+
     def test_list_comprehension(self):
         source = "def f(): [i for i in l]"
         source2 = "def f(): [i for i in l for j in l]"
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -671,13 +671,11 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket', 'array']}
 
     def setup_method(self, method):
-        w_HOST = space.wrap(self.HOST)
-        self.w_serv = space.appexec([w_socket, w_HOST],
+        w_HOST = self.space.wrap(self.HOST)
+        self.w_serv =self.space.appexec([w_socket, w_HOST],
             '''(_socket, HOST):
             serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
             serv.bind((HOST, 0))
@@ -687,7 +685,7 @@
 
     def teardown_method(self, method):
         if hasattr(self, 'w_serv'):
-            space.appexec([self.w_serv], '(serv): serv.close()')
+            self.space.appexec([self.w_serv], '(serv): serv.close()')
             self.w_serv = None
 
     def test_timeout(self):
@@ -803,8 +801,7 @@
 
 
 class AppTestErrno:
-    def setup_class(cls):
-        cls.space = space
+    spaceconfig = {'usemodules': ['_socket']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -3,8 +3,9 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
+    spaceconfig = {'usemodules': ['_vmprof', 'struct']}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
         cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1')))
         cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2')))
 
@@ -17,7 +18,7 @@
         import struct, sys, gc
 
         WORD = struct.calcsize('l')
-        
+
         def count(s):
             i = 0
             count = 0
@@ -44,7 +45,7 @@
                 else:
                     raise AssertionError(ord(s[i]))
             return count
-        
+
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -800,6 +800,21 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
+def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+    from rpython.rlib import rgil
+    # see "Handling of the GIL" above
+    assert cpyext_glob_tid_ptr[0] == 0
+    if pygilstate_release:
+        from pypy.module.cpyext import pystate
+        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
+    else:
+        unlock = gil_release or _gil_auto
+    if unlock:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
+
+
 def make_wrapper_second_level(space, argtypesw, restype,
                               result_kind, error_value, gil):
     from rpython.rlib import rgil
@@ -827,6 +842,7 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
+        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -839,7 +855,6 @@
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
-            from pypy.module.cpyext import pystate
             if cpyext_glob_tid_ptr[0] == tid:
                 cpyext_glob_tid_ptr[0] = 0
                 args += (pystate.PyGILState_LOCKED,)
@@ -850,6 +865,10 @@
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
+        if pygilstate_release:
+            gilstate = rffi.cast(lltype.Signed, args[-1])
+        else:
+            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -919,24 +938,13 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
+            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        # see "Handling of the GIL" above
-        assert cpyext_glob_tid_ptr[0] == 0
-        if pygilstate_release:
-            from pypy.module.cpyext import pystate
-            arg = rffi.cast(lltype.Signed, args[-1])
-            unlock = (arg == pystate.PyGILState_UNLOCKED)
-        else:
-            unlock = gil_release or _gil_auto
-        if unlock:
-            rgil.release()
-        else:
-            cpyext_glob_tid_ptr[0] = tid
-
+        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
@@ -1202,8 +1210,6 @@
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
-            if space.is_w(w_type, space.w_str):
-                pto.c_tp_itemsize = 1
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
@@ -1512,7 +1518,7 @@
     try:
         ll_libname = rffi.str2charp(path)
         try:
-            dll = rdynload.dlopen(ll_libname)
+            dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags)
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -6,7 +6,9 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref)
+    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
+    pyobj_has_w_obj)
+from pypy.objspace.std.bytesobject import W_BytesObject
 
 ##
 ## Implementation of PyStringObject
@@ -16,7 +18,7 @@
 ## -----------
 ##
 ## PyString_AsString() must return a (non-movable) pointer to the underlying
-## buffer, whereas pypy strings are movable.  C code may temporarily store
+## ob_sval, whereas pypy strings are movable.  C code may temporarily store
 ## this address and use it, as long as it owns a reference to the PyObject.
 ## There is no "release" function to specify that the pointer is not needed
 ## any more.
@@ -28,7 +30,7 @@
 ## --------
 ##
 ## PyStringObject contains two additional members: the ob_size and a pointer to a
-## char buffer; it may be NULL.
+## char ob_sval; it may be NULL.
 ##
 ## - A string allocated by pypy will be converted into a PyStringObject with a
 ##   NULL buffer.  The first time PyString_AsString() is called, memory is
@@ -41,6 +43,9 @@
 ##   the pypy string is created, and added to the global map of tracked
 ##   objects.  The buffer is then supposed to be immutable.
 ##
+##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
+##   there is no corresponding pypy object for the string
+##
 ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
 ##   similar object.
 ##
@@ -53,7 +58,7 @@
 PyStringObjectStruct = lltype.ForwardReference()
 PyStringObject = lltype.Ptr(PyStringObjectStruct)
 PyStringObjectFields = PyVarObjectFields + \
-    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP))
+    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char)))
 cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct)
 
 @bootstrap_function
@@ -69,44 +74,43 @@
 
 def new_empty_str(space, length):
     """
-    Allocate a PyStringObject and its buffer, but without a corresponding
-    interpreter object.  The buffer may be mutated, until string_realize() is
+    Allocate a PyStringObject and its ob_sval, but without a corresponding
+    interpreter object.  The ob_sval may be mutated, until string_realize() is
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_str.layout.typedef)
-    py_obj = typedescr.allocate(space, space.w_str)
+    py_obj = typedescr.allocate(space, space.w_str, length)
     py_str = rffi.cast(PyStringObject, py_obj)
-
-    buflen = length + 1
-    py_str.c_ob_size = length
-    py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+    py_str.c_ob_shash = -1
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
 def string_attach(space, py_obj, w_obj):
     """
-    Fills a newly allocated PyStringObject with the given string object. The
-    buffer must not be modified.
+    Copy RPython string object contents to a PyStringObject. The
+    c_ob_sval must not be modified.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    py_str.c_ob_size = len(space.str_w(w_obj))
-    py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
+    s = space.str_w(w_obj)
+    if py_str.c_ob_size  < len(s):
+        raise oefmt(space.w_ValueError,
+            "string_attach called on object with ob_size %d but trying to store %d",
+            py_str.c_ob_size, len(s)) 
+    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def string_realize(space, py_obj):
     """
-    Creates the string in the interpreter. The PyStringObject buffer must not
+    Creates the string in the interpreter. The PyStringObject ob_sval must not
     be modified after this call.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    if not py_str.c_buffer:
-        py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                    flavor='raw', zero=True)
-    s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
-    w_obj = space.wrap(s)
+    s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(W_BytesObject, w_type)
+    w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
@@ -116,9 +120,6 @@
 def string_dealloc(space, py_obj):
     """Frees allocated PyStringObject resources.
     """
-    py_str = rffi.cast(PyStringObject, py_obj)
-    if py_str.c_buffer:
-        lltype.free(py_str.c_buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -139,6 +140,9 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
+    return _PyString_AsString(space, ref)
+
+def _PyString_AsString(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         pass    # typecheck returned "ok" without forcing 'ref' at all
     elif not PyString_Check(space, ref):   # otherwise, use the alternate way
@@ -151,15 +155,23 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    return ref_str.c_buffer
+    if not pyobj_has_w_obj(ref):
+        # XXX Force the ref?
+        string_realize(space, ref)
+    return ref_str.c_ob_sval
+
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
+def PyString_AS_STRING(space, void_ref):
+    ref = rffi.cast(PyObject, void_ref)
+    # if no w_str is associated with this ref,
+    # return the c-level ptr as RW
+    if not pyobj_has_w_obj(ref):
+        py_str = rffi.cast(PyStringObject, ref)
+        return py_str.c_ob_sval
+    return _PyString_AsString(space, ref)
 
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
-def PyString_AsStringAndSize(space, ref, buffer, length):
+def PyString_AsStringAndSize(space, ref, data, length):
     if not PyString_Check(space, ref):
         from pypy.module.cpyext.unicodeobject import (
             PyUnicode_Check, _PyUnicode_AsDefaultEncodedString)
@@ -169,18 +181,16 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
+    if not pyobj_has_w_obj(ref):
+        # force the ref
+        string_realize(space, ref)
     ref_str = rffi.cast(PyStringObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    buffer[0] = ref_str.c_buffer
+    data[0] = ref_str.c_ob_sval
     if length:
         length[0] = ref_str.c_ob_size
     else:
         i = 0
-        while ref_str.c_buffer[i] != '\0':
+        while ref_str.c_ob_sval[i] != '\0':
             i += 1
         if i != ref_str.c_ob_size:
             raise oefmt(space.w_TypeError,
@@ -209,10 +219,10 @@
     set to NULL, a memory exception is set, and -1 is returned.
     """
     # XXX always create a new string so far
-    py_str = rffi.cast(PyStringObject, ref[0])
-    if not py_str.c_buffer:
+    if pyobj_has_w_obj(ref[0]):
         raise oefmt(space.w_SystemError,
                     "_PyString_Resize called on already created string")
+    py_str = rffi.cast(PyStringObject, ref[0])
     try:
         py_newstr = new_empty_str(space, newsize)
     except MemoryError:
@@ -224,7 +234,7 @@
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newstr.c_buffer[i] = py_str.c_buffer[i]
+        py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newstr)
     return 0
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.1-alpha0"
-#define PYPY_VERSION_NUM  0x05030100
+#define PYPY_VERSION "5.3.2-alpha0"
+#define PYPY_VERSION_NUM  0x05030200
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -1,5 +1,11 @@
 #include 
 
+#ifndef Py_PYMEM_H
+#define Py_PYMEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 #define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
 #define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
@@ -44,3 +50,9 @@
  */
 #define PyMem_Del               PyMem_Free
 #define PyMem_DEL               PyMem_FREE
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_PYMEM_H */
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -10,7 +10,6 @@
 #include 
 
 #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op))
-#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op))
 /*
 Type PyStringObject represents a character string.  An extra zero byte is
 reserved at the end to ensure it is zero-terminated, but a size is
@@ -41,12 +40,11 @@
     PyObject_VAR_HEAD
     long ob_shash;
     int ob_sstate;
-    char * buffer; /* change the name from cpython so all non-api c access is thwarted */
+    char ob_sval[1]; 
 
     /* Invariants 
-     * (not relevant in PyPy, all stringobjects are backed by a pypy object)
-     *     buffer contains space for 'ob_size+1' elements.
-     *     buffer[ob_size] == 0.
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
      *     ob_shash is the hash of the string or -1 if not computed yet.
      *     ob_sstate != 0 iff the string object is in stringobject.c's
      *       'interned' dictionary; in this case the two references
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -55,6 +55,7 @@
     if not PyFile_Check(space, w_p):
         raise oefmt(space.w_IOError, 'first argument must be an open file')
     assert isinstance(w_p, W_File)
+    w_p.stream.flush_buffers()
     try:
         fd = space.int_w(space.call_method(w_p, 'fileno'))
         mode = w_p.mode
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    INTERPLEVEL_API)
+    INTERPLEVEL_API, PyVarObject)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.objectobject import W_ObjectObject
@@ -47,13 +47,16 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount and w_type is not space.w_str:
+        if pytype.c_tp_itemsize:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
+        if pytype.c_tp_itemsize:
+            pyvarobj = rffi.cast(PyVarObject, pyobj)
+            pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
         #pyobj.c_ob_pypy_link should get assigned very quickly
         pyobj.c_ob_type = pytype
@@ -152,13 +155,18 @@
 class InvalidPointerException(Exception):
     pass
 
-def create_ref(space, w_obj, itemcount=0):
+def create_ref(space, w_obj):
     """
     Allocates a PyObject, and fills its fields with info from the given
     interpreter object.
     """
     w_type = space.type(w_obj)
+    pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
+    if pytype.c_tp_itemsize != 0:
+        itemcount = space.len_w(w_obj) # PyStringObject and subclasses
+    else:
+        itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)
     track_reference(space, py_obj, w_obj)
     #
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -207,6 +207,7 @@
 PyGILState_STATE = rffi.INT
 PyGILState_LOCKED = 0
 PyGILState_UNLOCKED = 1
+PyGILState_IGNORE = 2
 
 ExecutionContext.cpyext_gilstate_counter_noleave = 0
 
diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c
--- a/pypy/module/cpyext/src/stringobject.c
+++ b/pypy/module/cpyext/src/stringobject.c
@@ -107,7 +107,7 @@
     if (!string)
         return NULL;
 
-    s = PyString_AsString(string);
+    s = PyString_AS_STRING(string);
 
     for (f = format; *f; f++) {
         if (*f == '%') {
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import os
 import pytest
 
 def pytest_configure(config):
@@ -21,3 +21,14 @@
 def pytest_funcarg__api(request):
     return request.cls.api
 
+if os.name == 'nt':
+    @pytest.yield_fixture(autouse=True, scope='session')
+    def prevent_dialog_box():
+        """Do not open dreaded dialog box on segfault on Windows"""
+        import ctypes
+        SEM_NOGPFAULTERRORBOX = 0x0002  # From MSDN
+        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
+        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
+        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
+        yield
+        ctypes.windll.kernel32.SetErrorMode(old_err_mode)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -43,15 +43,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -195,7 +186,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -463,9 +454,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -508,12 +499,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/
diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/support.py
@@ -0,0 +1,68 @@
+import os
+import py
+from sys import platform
+
+if os.name != 'nt':
+    so_ext = 'so'
+else:
+    so_ext = 'dll'
+
+def c_compile(cfilenames, outputfilename,
+        compile_extra=None, link_extra=None,
+        include_dirs=None, libraries=None, library_dirs=None):
+    compile_extra = compile_extra or []
+    link_extra = link_extra or []
+    include_dirs = include_dirs or []
+    libraries = libraries or []
+    library_dirs = library_dirs or []
+    if platform == 'win32':
+        link_extra = link_extra + ['/DEBUG'] # generate .pdb file
+    if platform == 'darwin':
+        # support Fink & Darwinports
+        for s in ('/sw/', '/opt/local/'):
+            if (s + 'include' not in include_dirs
+                    and os.path.exists(s + 'include')):
+                include_dirs.append(s + 'include')
+            if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
+                library_dirs.append(s + 'lib')
+
+    outputfilename = py.path.local(outputfilename).new(ext=so_ext)
+    saved_environ = os.environ.copy()
+    try:
+        _build(
+            cfilenames, outputfilename,
+            compile_extra, link_extra,
+            include_dirs, libraries, library_dirs)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
+    return outputfilename
+
+def _build(cfilenames, outputfilename, compile_extra, link_extra,
+        include_dirs, libraries, library_dirs):
+    from distutils.ccompiler import new_compiler
+    from distutils import sysconfig
+    compiler = new_compiler(force=1)
+    sysconfig.customize_compiler(compiler) # XXX
+    objects = []
+    for cfile in cfilenames:
+        cfile = py.path.local(cfile)
+        old = cfile.dirpath().chdir()
+        try:
+            res = compiler.compile([cfile.basename],
+                include_dirs=include_dirs, extra_preargs=compile_extra)
+            assert len(res) == 1
+            cobjfile = py.path.local(res[0])
+            assert cobjfile.check()
+            objects.append(str(cobjfile))
+        finally:
+            old.chdir()
+
+    compiler.link_shared_object(
+        objects, str(outputfilename),
+        libraries=libraries,
+        extra_preargs=link_extra,
+        library_dirs=library_dirs)
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+
 class AppTestStringObject(AppTestCpythonExtensionBase):
     def test_basic(self):
         module = self.import_extension('foo', [
@@ -31,7 +32,7 @@
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
                      printf("tp_basicsize==%ld\\n",
-                            (long)s->ob_type->tp_basicsize); 
+                            (long)s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -53,7 +54,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyByteArray_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -84,11 +84,10 @@
             ("mutable", "METH_NOARGS",
              """
                 PyObject *base;
-                char * p_str;
                 base = PyByteArray_FromStringAndSize("test", 10);
                 if (PyByteArray_GET_SIZE(base) != 10)
                     return PyLong_FromLong(-PyByteArray_GET_SIZE(base));
-                memcpy(PyByteArray_AS_STRING(base), "works", 6); 
+                memcpy(PyByteArray_AS_STRING(base), "works", 6);
                 Py_INCREF(base);
                 return base;
              """),
@@ -115,6 +114,7 @@
         assert s == 'test'
 
     def test_manipulations(self):
+        import sys
         module = self.import_extension('foo', [
             ("bytearray_from_string", "METH_VARARGS",
              '''
@@ -141,9 +141,9 @@
             ("concat", "METH_VARARGS",
              """
                 PyObject * ret, *right, *left;
-                PyObject *ba1, *ba2; 
+                PyObject *ba1, *ba2;
                 if (!PyArg_ParseTuple(args, "OO", &left, &right)) {
-                    return PyString_FromString("parse failed"); 
+                    return PyString_FromString("parse failed");
                 }
                 ba1 = PyByteArray_FromObject(left);
                 ba2 = PyByteArray_FromObject(right);
@@ -157,7 +157,9 @@
              """)])
         assert module.bytearray_from_string("huheduwe") == "huhe"
         assert module.str_from_bytearray(bytearray('abc')) == 'abc'
-        raises(ValueError, module.str_from_bytearray, 4.0)
+        if '__pypy__' in sys.builtin_module_names:
+            # CPython only makes an assert.
+            raises(ValueError, module.str_from_bytearray, 4.0)
         ret = module.concat('abc', 'def')
         assert ret == 'abcdef'
         assert not isinstance(ret, str)
@@ -171,9 +173,9 @@
              PyObject *obj, *ba;
              int newsize, oldsize, ret;
              if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) {
-                 return PyString_FromString("parse failed"); 
+                 return PyString_FromString("parse failed");
              }
-             
+
              ba = PyByteArray_FromObject(obj);
              if (ba == NULL)
                  return NULL;
@@ -187,7 +189,7 @@
              {
                   printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize);
                   return NULL;
-             } 
+             }
              return ba;
              '''
             )])
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -25,14 +25,13 @@
             ("test_Size", "METH_NOARGS",
              """
                  PyObject* s = PyString_FromString("Hello world");
-                 int result = 0;
+                 int result;
                  size_t expected_size;
 
-                 if(PyString_Size(s) == 11) {
-                     result = 1;
-                 }
+                 result = PyString_Size(s);
+
                  #ifdef PYPY_VERSION
-                    expected_size = sizeof(void*)*7;
+                    expected_size = 48;
                  #elif defined Py_DEBUG
                     expected_size = 53;
                  #else
@@ -40,16 +39,16 @@
                  #endif
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
-                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); 
+                     printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
-                 return PyBool_FromLong(result);
+                 return PyLong_FromLong(result);
              """),
             ("test_Size_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyString_Size(f);
+                 PyString_Size(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -60,7 +59,7 @@
              """)], prologue='#include ')
         assert module.get_hello1() == 'Hello world'
         assert module.get_hello2() == 'Hello world'
-        assert module.test_Size()
+        assert module.test_Size() == 11
         raises(TypeError, module.test_Size_exception)
 
         assert module.test_is_string("")
@@ -72,7 +71,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyString_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -81,7 +79,7 @@
                  if (t == NULL)
                     return NULL;
                  Py_DECREF(t);
-                 c = PyString_AsString(s);
+                 c = PyString_AS_STRING(s);
                  c[0] = 'a';
                  c[1] = 'b';
                  c[2] = 0;
@@ -100,7 +98,6 @@
                 PyObject *base;
                 PyTypeObject * type;
                 PyStringObject *obj;
-                char * p_str;
                 base = PyString_FromString("test");
                 if (PyString_GET_SIZE(base) != 4)
                     return PyLong_FromLong(-PyString_GET_SIZE(base));
@@ -110,14 +107,22 @@
                 obj = (PyStringObject*)type->tp_alloc(type, 10);
                 if (PyString_GET_SIZE(obj) != 10)
                     return PyLong_FromLong(PyString_GET_SIZE(obj));
-                /* cannot work, there is only RO access
-                memcpy(PyString_AS_STRING(obj), "works", 6); */
+                /* cannot work, there is only RO access */
+                memcpy(PyString_AS_STRING(obj), "works", 6);
                 Py_INCREF(obj);
                 return (PyObject*)obj;
              """),
+            ('alloc_rw', "METH_NOARGS",
+             '''
+                PyObject *obj = _PyObject_NewVar(&PyString_Type, 10);
+                memcpy(PyString_AS_STRING(obj), "works", 6);
+                return (PyObject*)obj;
+             '''),
             ])
+        s = module.alloc_rw()
+        assert s == 'works' + '\x00' * 5
         s = module.tpalloc()
-        assert s == '\x00' * 10
+        assert s == 'works' + '\x00' * 5
 
     def test_AsString(self):
         module = self.import_extension('foo', [
@@ -312,17 +317,17 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyStringObject*)obj)->ob_shash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ("test_sstate", "METH_NOARGS",
              '''
                 PyObject *s = PyString_FromString("xyz");
-                int sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*int sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 PyString_InternInPlace(&s);
-                sstate = ((PyStringObject*)s)->ob_sstate;
-                /*printf("sstate now %d\\n", sstate);*/
+                /*sstate = ((PyStringObject*)s)->ob_sstate;
+                printf("sstate now %d\\n", sstate);*/
                 Py_DECREF(s);
                 return PyBool_FromLong(1);
              '''),
@@ -332,27 +337,124 @@
         # doesn't really test, but if printf is enabled will prove sstate
         assert module.test_sstate()
 
+    def test_subclass(self):
+        # taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        module = self.import_extension('bar', [
+            ("newsubstr", "METH_O",
+             """
+                PyObject * obj;
+                char * data;
+                int len;
+                PyType_Ready(&PyStringArrType_Type);
+
+                data = PyString_AS_STRING(args);
+                len = PyString_GET_SIZE(args);
+                if (data == NULL || len < 1)
+                    Py_RETURN_NONE;
+                obj = PyArray_Scalar(data, len);
+                return obj;
+             """),
+            ], prologue="""
+                #include 
+                PyTypeObject PyStringArrType_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,                            /* ob_size */
+                    "bar.string_",                /* tp_name*/
+                    sizeof(PyStringObject), /* tp_basicsize*/
+                    0                             /* tp_itemsize */
+                    };
+
+                    static PyObject *
+                    stringtype_repr(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    static PyObject *
+                    stringtype_str(PyObject *self)
+                    {
+                        const char *dptr, *ip;
+                        int len;
+                        PyObject *new;
+
+                        ip = dptr = PyString_AS_STRING(self);
+                        len = PyString_GET_SIZE(self);
+                        dptr += len-1;
+                        while(len > 0 && *dptr-- == 0) {
+                            len--;
+                        }
+                        new = PyString_FromStringAndSize(ip, len);
+                        if (new == NULL) {
+                            return PyString_FromString("");
+                        }
+                        return new;
+                    }
+
+                    PyObject *
+                    PyArray_Scalar(char *data, int n)
+                    {
+                        PyTypeObject *type = &PyStringArrType_Type;
+                        PyObject *obj;
+                        void *destptr;
+                        int itemsize = n;
+                        obj = type->tp_alloc(type, itemsize);
+                        if (obj == NULL) {
+                            return NULL;
+                        }
+                        destptr = PyString_AS_STRING(obj);
+                        ((PyStringObject *)obj)->ob_shash = -1;
+                        memcpy(destptr, data, itemsize);
+                        return obj;
+                    }
+            """, more_init = '''
+                PyStringArrType_Type.tp_alloc = NULL;
+                PyStringArrType_Type.tp_free = NULL;
+
+                PyStringArrType_Type.tp_repr = stringtype_repr;
+                PyStringArrType_Type.tp_str = stringtype_str;
+                PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
+                PyStringArrType_Type.tp_itemsize = sizeof(char);
+                PyStringArrType_Type.tp_base = &PyString_Type;
+            ''')
+
+        a = module.newsubstr('abc')
+        assert type(a).__name__ == 'string_'
+        assert a == 'abc'
 
 class TestString(BaseApiTest):
     def test_string_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_buffer[0] = 'a'
-        py_str.c_buffer[1] = 'b'
-        py_str.c_buffer[2] = 'c'
+        py_str.c_ob_sval[0] = 'a'
+        py_str.c_ob_sval[1] = 'b'
+        py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 3)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[3] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyString_Resize(ar, 10)
         py_str = rffi.cast(PyStringObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[10] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,8 +7,6 @@
 from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import platform
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
@@ -18,20 +16,7 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-def setup_module(module):
-    if os.name == 'nt':
-        # Do not open dreaded dialog box on segfault
-        import ctypes
-        SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
-        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
-        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
-        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
-        module.old_err_mode = old_err_mode
-
-def teardown_module(module):
-    if os.name == 'nt':
-        import ctypes
-        ctypes.windll.kernel32.SetErrorMode(module.old_err_mode)
+from .support import c_compile
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -46,7 +31,30 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
-def compile_extension_module(space, modname, include_dirs=[], **kwds):
+def convert_sources_to_files(sources, dirname):
+    files = []
+    for i, source in enumerate(sources):
+        filename = dirname / ('source_%d.c' % i)
+        with filename.open('w') as f:
+            f.write(str(source))
+        files.append(filename)
+    return files
+
+def create_so(modname, include_dirs, source_strings=None, source_files=None,
+        compile_extra=None, link_extra=None, libraries=None):
+    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+    if source_strings:
+        assert not source_files
+        files = convert_sources_to_files(source_strings, dirname)
+        source_files = files
+    soname = c_compile(source_files, outputfilename=str(dirname/modname),
+        compile_extra=compile_extra, link_extra=link_extra,
+        include_dirs=include_dirs,
+        libraries=libraries)
+    return soname
+
+def compile_extension_module(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -60,38 +68,36 @@
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        libraries = [api_library]
         # '%s' undefined; assuming extern returning int
-        kwds["compile_extra"] = ["/we4013"]
+        compile_extra = ["/we4013"]
         # prevent linking with PythonXX.lib
         w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
                               (space.int_w(w_maj), space.int_w(w_min))]
-    elif sys.platform == 'darwin':
-        kwds["link_files"] = [str(api_library + '.dylib')]
     else:
-        kwds["link_files"] = [str(api_library + '.so')]
+        libraries = []
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
-            kwds["link_extra"]=["-g"]
+            compile_extra = ["-Werror", "-g", "-O0", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs=api.include_dirs + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=api.include_dirs + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra,
+            libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
     pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
     soname.rename(pydname)
     return str(pydname)
 
-def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds):
+def compile_extension_module_applevel(space, modname, include_dirs=[],
+        source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -103,24 +109,23 @@
     build the module (so specify your source with one of those).
     """
     if sys.platform == 'win32':
-        kwds["compile_extra"] = ["/we4013"]
-        kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
+        compile_extra = ["/we4013"]
+        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
     elif sys.platform == 'darwin':
+        compile_extra = link_extra = None
         pass
     elif sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"]
+        compile_extra = [
+            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
+        link_extra = None
 
     modname = modname.split('.')[-1]
-    eci = ExternalCompilationInfo(
-        include_dirs = [space.include_dir] + include_dirs,
-        **kwds
-        )
-    eci = eci.convert_sources_to_files()
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    soname = platform.platform.compile(
-        [], eci,
-        outputfilename=str(dirname/modname),
-        standalone=False)
+    soname = create_so(modname,
+            include_dirs=[space.include_dir] + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=compile_extra,
+            link_extra=link_extra)
     return str(soname)
 
 def freeze_refcnts(self):
@@ -285,8 +290,8 @@
                 separate_module_sources = []
             pydname = self.compile_extension_module(
                 space, name,
-                separate_module_files=separate_module_files,
-                separate_module_sources=separate_module_sources)
+                source_files=separate_module_files,
+                source_strings=separate_module_sources)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -315,6 +320,11 @@
                 /* fix for cpython 2.7 Python.h if running tests with -A
                    since pypy compiles with -fvisibility-hidden */
                 #undef PyMODINIT_FUNC
+                #ifdef __GNUC__
+                #  define RPY_EXPORTED extern __attribute__((visibility("default")))
+                #else
+                #  define RPY_EXPORTED extern __declspec(dllexport)
+                #endif
                 #define PyMODINIT_FUNC RPY_EXPORTED void
 
                 %(body)s
@@ -326,16 +336,16 @@
                 """ % dict(name=name, init=init, body=body,
                            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
                                             if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(separate_module_sources=[code])
+                kwds = dict(source_strings=[code])
             else:
                 assert not PY_SSIZE_T_CLEAN
                 if filename is None:
                     filename = name
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(separate_module_files=[filename])
-            kwds['include_dirs'] = include_dirs
-            mod = self.compile_extension_module(space, name, **kwds)
+                kwds = dict(source_files=[filename])
+            mod = self.compile_extension_module(space, name,
+                    include_dirs=include_dirs, **kwds)
 
             if load_it:
                 if self.runappdirect:
@@ -975,7 +985,7 @@
             ('bar', 'METH_NOARGS',
              '''
              /* reuse a name that is #defined in structmember.h */
-             int RO;
+             int RO = 0; (void)RO;
              Py_RETURN_NONE;
              '''
              ),
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -142,7 +142,7 @@
                     2000, 6, 6, 6, 6, 6, 6, Py_None,
                     PyDateTimeAPI->DateTimeType);
              """),
-        ])
+        ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.new_date() == datetime.date(2000, 6, 6)
         assert module.new_time() == datetime.time(6, 6, 6, 6)
@@ -241,6 +241,9 @@
                  PyObject* obj = PyDelta_FromDSU(6, 6, 6);
                  PyDateTime_Delta* delta = (PyDateTime_Delta*)obj;
 
+#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000
+                 // These macros are only defined in CPython 3.x and PyPy.
+                 // See: http://bugs.python.org/issue13727
                  PyDateTime_DELTA_GET_DAYS(obj);
                  PyDateTime_DELTA_GET_DAYS(delta);
 
@@ -249,10 +252,10 @@
 
                  PyDateTime_DELTA_GET_MICROSECONDS(obj);
                  PyDateTime_DELTA_GET_MICROSECONDS(delta);
-
+#endif
                  return obj;
              """),
-            ])
+            ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.test_date_macros() == datetime.date(2000, 6, 6)
         assert module.test_datetime_macros() == datetime.datetime(
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py
--- a/pypy/module/cpyext/test/test_frameobject.py
+++ b/pypy/module/cpyext/test/test_frameobject.py
@@ -13,7 +13,7 @@
                  PyObject *empty_string = PyString_FromString("");
                  PyObject *empty_tuple = PyTuple_New(0);
                  PyCodeObject *py_code;
-                 PyFrameObject *py_frame;
+                 PyFrameObject *py_frame = NULL;
 
                  py_code = PyCode_New(
                      0,            /*int argcount,*/
@@ -75,7 +75,7 @@
              """
                  int check;
                  PyObject *type, *value, *tb;
-                 PyObject *ret = PyRun_String("XXX", Py_eval_input, 
+                 PyObject *ret = PyRun_String("XXX", Py_eval_input,
                                               Py_None, Py_None);
                  if (ret) {
                      Py_DECREF(ret);
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -149,7 +149,6 @@
         pybuffer = self.import_parser(
             '''
             Py_buffer buf1, buf2, buf3;
-            PyObject *result;
             if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) {
                 return NULL;
             }
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.descriptor import get_dtype_cache
-import pypy.module.micronumpy.constants as NPY 
+import pypy.module.micronumpy.constants as NPY
 
 def scalar(space):
     dtype = get_dtype_cache(space).w_float64dtype
@@ -237,7 +237,7 @@
             except:
                 skip('numpy not importable')
         else:
-            numpy_incl = os.path.abspath(os.path.dirname(__file__) + 
+            numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
             assert os.path.exists(numpy_incl)
             cls.w_numpy_include = cls.space.wrap([numpy_incl])
@@ -273,7 +273,7 @@
                 {
                     /* Should have failed */
                     Py_DECREF(obj1);
-                    return NULL; 
+                    return NULL;
                 }
                 return obj1;
                 '''
@@ -300,14 +300,14 @@
                 ),
                 ("test_DescrFromType", "METH_O",
                 """
-                    Signed typenum = PyInt_AsLong(args);
+                    long typenum = PyInt_AsLong(args);
                     return PyArray_DescrFromType(typenum);
                 """
                 ),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -315,7 +315,7 @@
                     #define PyArray_FromObject _PyArray_FromObject
                     #define PyArray_FromAny _PyArray_FromAny
                 #endif
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -349,14 +349,14 @@
                     Py_INCREF(obj);
                     return obj;
                 '''),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
-                ''', 
+                ''',
                     more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
@@ -403,14 +403,14 @@
                 void *array_data[] = {NULL, NULL};
                 return PyUFunc_FromFuncAndDataAndSignature(funcs,
                                     array_data, types, 1, 1, 1, PyUFunc_None,
-                                    "float_3x3", 
-                                    "a ufunc that tests a more complicated signature", 
+                                    "float_3x3",
+                                    "a ufunc that tests a more complicated signature",
                                     0, "(m,m)->(m,m)");
                 """),
-                ], include_dirs=self.numpy_include, 
+                ], include_dirs=self.numpy_include,
                    prologue='''
                 #ifdef PYPY_VERSION
-                    #include     
+                    #include 
                 #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include 
@@ -480,7 +480,7 @@
                             res += +10;
                     *((float *)args[1]) = res;
                 };
-                            
+
                 ''',  more_init = '''
                 #ifndef PYPY_VERSION
                     import_array();
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -127,12 +127,12 @@
         test_compare(1, 2)
         test_compare(2, 2)
         test_compare('2', '1')
-        
+
         w_i = space.wrap(1)
         assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1
         assert api.PyErr_Occurred() is space.w_SystemError
         api.PyErr_Clear()
-        
+
     def test_IsInstance(self, space, api):
         assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1
         assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0
@@ -165,7 +165,7 @@
             return File""")
         w_f = space.call_function(w_File)
         assert api.PyObject_AsFileDescriptor(w_f) == 42
-    
+
     def test_hash(self, space, api):
         assert api.PyObject_Hash(space.wrap(72)) == 72
         assert api.PyObject_Hash(space.wrap(-1)) == -1
@@ -250,7 +250,7 @@
                  if (copy != orig)
                      PyObject_Free(copy);
                  PyObject_Free(orig);
-                 return ret;  
+                 return ret;
              """)])
         x = module.realloctest()
         assert x == 'hello world\x00'
@@ -425,7 +425,6 @@
                  """
     Py_buffer buf;
     PyObject *str = PyString_FromString("hello, world.");
-    PyObject *result;
 
     if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13,
                           1, PyBUF_WRITABLE)) {
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,7 @@
+from pypy.conftest import option
 from pypy.module.cpyext.api import fopen, fclose, fwrite
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
@@ -111,3 +113,34 @@
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == " 1 23\n"
+
+
+class AppTestPyFile(AppTestCpythonExtensionBase):
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    def test_file_tell(self):
+        module = self.import_extension('foo', [
+            ("get_c_tell", "METH_O",
+             """
+                FILE * fp = PyFile_AsFile(args);
+                if (fp == NULL)
+                    return PyLong_FromLong(0);
+                return PyLong_FromLong(ftell(fp));
+             """),
+            ])
+        filename = self.udir + "/_test_file"
+        with open(filename, 'w') as fid:
+            fid.write('3' * 122)
+        with open(filename, 'r') as fid:
+            s = fid.read(80)
+            t_py = fid.tell()
+            assert t_py == 80
+            t_c = module.get_c_tell(fid)
+        assert t_c == t_py
+
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -130,7 +130,7 @@
         module = self.import_extension('foo', [
             ("run", "METH_NOARGS",
              """
-                long prev, next;
+                long prev;
                 PyObject *t = PyTuple_New(1);
                 prev = Py_True->ob_refcnt;
                 Py_INCREF(Py_True);
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -5,8 +5,6 @@
 from pypy.module.cpyext.pyobject import make_ref, from_ref
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
-import sys
-
 class AppTestTypeObject(AppTestCpythonExtensionBase):
     def test_typeobject(self):
         import sys
@@ -737,7 +735,6 @@
              """
                 IntLikeObject *intObj;
                 int intval;
-                PyObject *name;
 
                 if (!PyArg_ParseTuple(args, "i", &intval))
                     return NULL;
@@ -897,7 +894,7 @@
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
-        # on cpython, the size changes (6 bytes added)
+        import sys
         module = self.import_module(name='foo')
         size = module.size_of_instances(module.fooType)
         class f1(object):
@@ -907,7 +904,11 @@
         class bar(f1, f2):
             pass
         assert bar.__base__ is f2
-        assert module.size_of_instances(bar) == size
+        # On cpython, the size changes.
+        if '__pypy__' in sys.builtin_module_names:
+            assert module.size_of_instances(bar) == size
+        else:
+            assert module.size_of_instances(bar) >= size
 
     def test_app_cant_subclass_two_types(self):
         module = self.import_module(name='foo')
@@ -1058,7 +1059,6 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                PyObject *obj;
                 FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
                 FooType_Type.tp_base = &PyType_Type;
                 if (PyType_Ready(&FooType_Type) < 0) return NULL;
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -35,7 +35,7 @@
             ("test_GetSize_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyUnicode_GetSize(f);
+                 PyUnicode_GetSize(f);
 
                  Py_DECREF(f);
                  return NULL;
@@ -57,7 +57,6 @@
              """
                  PyObject *s, *t;
                  Py_UNICODE* c;
-                 Py_ssize_t len;
 
                  s = PyUnicode_FromUnicode(NULL, 4);
                  if (s == NULL)
@@ -84,7 +83,7 @@
              '''
                 PyObject* obj = (PyTuple_GetItem(args, 0));
                 long hash = ((PyUnicodeObject*)obj)->hash;
-                return PyLong_FromLong(hash);  
+                return PyLong_FromLong(hash);
              '''
              ),
             ])
@@ -234,13 +233,13 @@
         w_res = api.PyUnicode_AsUTF8String(w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\tm'
-    
+
     def test_decode_utf8(self, space, api):
         u = rffi.str2charp(u'sp\x134m'.encode("utf-8"))
         w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp\x134m'
-        
+
         w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
@@ -405,7 +404,7 @@
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
         result = api.PyUnicode_AsASCIIString(w_ustr)
-        
+
         assert space.eq_w(space.wrap(ustr), result)
 
         w_ustr = space.wrap(u"abcd\xe9f")
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -657,6 +657,8 @@
         pto.c_tp_dealloc = llhelper(
             subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
+    if space.is_w(w_type, space.w_str):
+        pto.c_tp_itemsize = 1
     # buffer protocol
     setup_buffer_procs(space, w_type, pto)
 
@@ -695,6 +697,8 @@
     if pto.c_tp_base:
         if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize:
             pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize
+        if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
+            pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize

From pypy.commits at gmail.com  Tue Jun 21 09:33:43 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 06:33:43 -0700 (PDT)
Subject: [pypy-commit] pypy new-jit-log: fixing tests
Message-ID: <57694237.e643c20a.184ee.7bc4@mx.google.com>

Author: Richard Plangger 
Branch: new-jit-log
Changeset: r85303:4566282be496
Date: 2016-06-21 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4566282be496/

Log:	fixing tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -57,7 +57,10 @@
         cmdline.append(str(self.filepath))
         #
         env = os.environ.copy()
+        # TODO old logging system
         env['PYPYLOG'] = self.log_string + ':' + str(logfile)
+        jitlogfile = str(logfile) + '.jlog'
+        env['JITLOG'] = str(jitlogfile)
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
@@ -84,6 +87,7 @@
         log = Log(rawtraces)
         log.result = eval(stdout)
         log.logfile = str(logfile)
+        log.jitlogfile = jitlogfile
         #
         summaries  = logparser.extract_category(rawlog, 'jit-summary')
         if len(summaries) > 0:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
--- a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
@@ -8,6 +8,7 @@
         mangle_descr)
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
+class TestIntegrationJitLog(BaseTestPyPyC):
 
 class TestLogParser(BaseTestPyPyC):
     log_string = 'jit-log-opt,jit-backend'
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -335,13 +335,8 @@
     @specialize.argtype(1)
     def _inject_debugging_code(self, looptoken, operations, tp, number):
         if self._debug:
-            s = 0
-            for op in operations:
-                s += op.getopnum()
-
             newoperations = []
-            self._append_debugging_code(newoperations, tp, number,
-                                        None)
+            self._append_debugging_code(newoperations, tp, number, None)
             for op in operations:
                 newoperations.append(op)
                 if op.getopnum() == rop.LABEL:
@@ -371,7 +366,7 @@
         LOOP_RUN_COUNTERS.append(struct)
         return struct
 
-    def finish_once(self, jitlog):
+    def finish_once(self):
         if self._debug:
             # TODO remove the old logging system when jitlog is complete
             debug_start('jit-backend-counts')
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -113,7 +113,7 @@
                      unique_id=0, log=True, name='', logger=None):
         return self.assembler.assemble_loop(jd_id, unique_id, logger, name,
                                             inputargs, operations,
-                                            looptoken, log=log)
+                                            looptoken, log)
 
     def stitch_bridge(self, faildescr, target):
         self.assembler.stitch_bridge(faildescr, target)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -66,8 +66,8 @@
         self.assembler.setup_once()
 
     @rgc.no_release_gil
-    def finish_once(self, jitlog=None):
-        self.assembler.finish_once(jitlog)
+    def finish_once(self):
+        self.assembler.finish_once()
         self.profile_agent.shutdown()
 
     def dump_loop_token(self, looptoken):
diff --git a/rpython/jit/backend/x86/test/test_jitlog.py b/rpython/jit/backend/x86/test/test_jitlog.py
--- a/rpython/jit/backend/x86/test/test_jitlog.py
+++ b/rpython/jit/backend/x86/test/test_jitlog.py
@@ -30,7 +30,7 @@
         assert os.path.exists(file.strpath)
         with file.open('rb') as f:
             # check the file header
-            assert f.read(3) == chr(jl.MARK_JITLOG_HEADER) + JITLOG_VERSION_16BIT_LE
+            assert f.read(3) == jl.MARK_JITLOG_HEADER + JITLOG_VERSION_16BIT_LE
             assert len(f.read()) > 0
 
     def test_env(self, monkeypatch, tmpdir):
@@ -41,7 +41,7 @@
         assert os.path.exists(file.strpath)
         with file.open('rb') as fd:
             # check the file header
-            assert fd.read(3) == chr(jl.MARK_JITLOG_HEADER) + JITLOG_VERSION_16BIT_LE
+            assert fd.read(3) == jl.MARK_JITLOG_HEADER + JITLOG_VERSION_16BIT_LE
             assert len(fd.read()) > 0
 
     def test_version(self, monkeypatch, tmpdir):
@@ -53,7 +53,7 @@
         assert os.path.exists(file.strpath)
         with file.open('rb') as fd:
             # check the file header
-            assert fd.read(3) == chr(jl.MARK_JITLOG_HEADER) + '\xff\xfe'
+            assert fd.read(3) == jl.MARK_JITLOG_HEADER + '\xff\xfe'
             assert len(fd.read()) > 0
 
     def test_version(self, monkeypatch, tmpdir):
@@ -65,7 +65,7 @@
         assert os.path.exists(file.strpath)
         with file.open('rb') as fd:
             # check the file header
-            assert fd.read(3) == chr(jl.MARK_JITLOG_HEADER) + '\xff\xfe'
+            assert fd.read(3) == jl.MARK_JITLOG_HEADER + '\xff\xfe'
             assert len(fd.read()) > 0
 
     def run_sample_loop(self, func, myjitdriver = None):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -1070,8 +1070,7 @@
         def finish():
             if self.metainterp_sd.profiler.initialized:
                 self.metainterp_sd.profiler.finish()
-            jitlog = self.metainterp_sd.jitlog
-            self.metainterp_sd.cpu.finish_once(jitlog)
+            self.metainterp_sd.cpu.finish_once()
 
         if self.cpu.translate_support_code:
             call_final_function(self.translator, finish,
diff --git a/rpython/rlib/jitlog.py b/rpython/rlib/jitlog.py
--- a/rpython/rlib/jitlog.py
+++ b/rpython/rlib/jitlog.py
@@ -269,6 +269,8 @@
         self._write_marked(MARK_START_TRACE, ''.join(content))
 
     def trace_aborted(self):
+        if not self.cintf.jitlog_enabled():
+            return
         self._write_marked(MARK_ABORT_TRACE, encode_le_64bit(self.trace_id))
 
     def _write_marked(self, mark, line):
@@ -283,7 +285,6 @@
         if not self.cintf.jitlog_enabled():
             return EMPTY_TRACE_LOG
         assert self.metainterp_sd is not None
-        assert isinstance(tag, int)
         if memo is None:
             memo = {}
         return LogTrace(tag, memo, self.metainterp_sd, mc, self)
diff --git a/rpython/rlib/test/test_jitlog.py b/rpython/rlib/test/test_jitlog.py
--- a/rpython/rlib/test/test_jitlog.py
+++ b/rpython/rlib/test/test_jitlog.py
@@ -12,7 +12,7 @@
         self.values = []
 
     def _write_marked(self, id, text):
-        self.values.append(chr(id) + text)
+        self.values.append(id + text)
 
 def _get_location(greenkey_list):
     assert len(greenkey_list) == 0
@@ -56,12 +56,12 @@
         fd.close()
         logger.finish()
         binary = file.read()
-        assert binary == chr(jl.MARK_START_TRACE) + jl.encode_le_addr(0) + \
+        assert binary == (jl.MARK_START_TRACE) + jl.encode_le_addr(0) + \
                          jl.encode_str('loop') + jl.encode_le_addr(0) + \
-                         chr(jl.MARK_TRACE) + jl.encode_le_addr(0) + \
-                         chr(jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
-                         chr(jl.MARK_INIT_MERGE_POINT) + b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
-                         chr(jl.MARK_MERGE_POINT) + \
+                         (jl.MARK_TRACE) + jl.encode_le_addr(0) + \
+                         (jl.MARK_INPUT_ARGS) + jl.encode_str('') + \
+                         (jl.MARK_INIT_MERGE_POINT) + b'\x05\x00\x01s\x00i\x08s\x00i\x10s' + \
+                         (jl.MARK_MERGE_POINT) + \
                          b'\xff' + encode_str('/home/pypy/jit.py') + \
                          b'\x00' + encode_le_64bit(0) + \
                          b'\xff' + encode_str('enclosed') + \
@@ -78,12 +78,12 @@
         #
         result = jl.encode_merge_point(fakelog, compressor, [jl.StringValue(0x0,'s','hello')])
         assert result == b"\xef"
-        assert fakelog.values == [chr(jl.MARK_COMMON_PREFIX) + "\x00\x05\x00\x00\x00hello"]
+        assert fakelog.values == [(jl.MARK_COMMON_PREFIX) + "\x00\x05\x00\x00\x00hello"]
         #
         fakelog.values = []
         result = jl.encode_merge_point(fakelog, compressor, [jl.StringValue(0x0,'s','heiter')])
         assert result == b"\x00\x04\x00\x00\x00iter"
-        assert fakelog.values == [chr(jl.MARK_COMMON_PREFIX) + "\x00\x02\x00\x00\x00he"]
+        assert fakelog.values == [(jl.MARK_COMMON_PREFIX) + "\x00\x02\x00\x00\x00he"]
         #
         fakelog.values = []
         result = jl.encode_merge_point(fakelog, compressor, [jl.StringValue(0x0,'s','heute')])
@@ -98,7 +98,7 @@
         fakelog.values = []
         result = jl.encode_merge_point(fakelog, compressor, [jl.StringValue(0x0,'s','welle')])
         assert result == b"\x00\x02\x00\x00\x00le"
-        assert fakelog.values == [chr(jl.MARK_COMMON_PREFIX) + "\x00\x03\x00\x00\x00wel"]
+        assert fakelog.values == [(jl.MARK_COMMON_PREFIX) + "\x00\x03\x00\x00\x00wel"]
 
     def test_common_prefix_func(self):
         assert jl.commonprefix("","") == ""

From pypy.commits at gmail.com  Tue Jun 21 09:39:57 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 06:39:57 -0700 (PDT)
Subject: [pypy-commit] pypy new-jit-log: added function to access loop run
 counters of the assembler (instead of accessing the list directly)
Message-ID: <576943ad.cbc71c0a.55f91.1634@mx.google.com>

Author: Richard Plangger 
Branch: new-jit-log
Changeset: r85304:d37d1456081b
Date: 2016-06-21 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/d37d1456081b/

Log:	added function to access loop run counters of the assembler (instead
	of accessing the list directly)

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -332,6 +332,9 @@
         # Here we join Path A and Path B again
         self._call_assembler_patch_jmp(jmp_location)
 
+    def get_loop_run_counters(self, index):
+        return LOOP_RUN_COUNTERS[index]
+
     @specialize.argtype(1)
     def _inject_debugging_code(self, looptoken, operations, tp, number):
         if self._debug:
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -590,11 +590,11 @@
             self.cpu.compile_loop(ops.inputargs, ops.operations, looptoken)
             self.cpu.execute_token(looptoken, 0)
             # check debugging info
-            struct = self.cpu.assembler.loop_run_counters[0]
+            struct = self.cpu.assembler.get_loop_run_counters(0)
             assert struct.i == 1
-            struct = self.cpu.assembler.loop_run_counters[1]
+            struct = self.cpu.assembler.get_loop_run_counters(1)
             assert struct.i == 1
-            struct = self.cpu.assembler.loop_run_counters[2]
+            struct = self.cpu.assembler.get_loop_run_counters(2)
             assert struct.i == 9
             self.cpu.finish_once()
         finally:

From pypy.commits at gmail.com  Tue Jun 21 10:49:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 07:49:10 -0700 (PDT)
Subject: [pypy-commit] pypy new-jit-log: removed syntax error
Message-ID: <576953e6.c3a9c20a.f1c32.ffffe58d@mx.google.com>

Author: Richard Plangger 
Branch: new-jit-log
Changeset: r85305:6fdd6f3fb263
Date: 2016-06-21 16:46 +0200
http://bitbucket.org/pypy/pypy/changeset/6fdd6f3fb263/

Log:	removed syntax error

diff --git a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
--- a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
@@ -8,8 +8,6 @@
         mangle_descr)
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
-class TestIntegrationJitLog(BaseTestPyPyC):
-
 class TestLogParser(BaseTestPyPyC):
     log_string = 'jit-log-opt,jit-backend'
 

From pypy.commits at gmail.com  Tue Jun 21 10:53:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Jun 2016 07:53:41 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress: tracking live
 objects
Message-ID: <576954f5.42431c0a.15167.3254@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85306:1c3f6914ae95
Date: 2016-06-21 16:54 +0200
http://bitbucket.org/pypy/pypy/changeset/1c3f6914ae95/

Log:	in-progress: tracking live objects

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -17,6 +17,7 @@
 CMD_MOREINFO    = 5
 ANSWER_TEXT     = 20
 ANSWER_MOREINFO = 21
+ANSWER_NEXTNID  = 22
 
 
 def stop_point():
@@ -31,6 +32,9 @@
 def register_debug_command(command, lambda_func):
     """Register the extra RPython-implemented debug command."""
 
+def register_allocation_command(lambda_func):
+    """Register the extra RPython-implemented callback for allocation."""
+
 def send_answer(cmd, arg1=0, arg2=0, arg3=0, extra=""):
     """For RPython debug commands: writes an answer block to stdout"""
     llop.revdb_send_answer(lltype.Void, cmd, arg1, arg2, arg3, extra)
@@ -38,6 +42,9 @@
 def send_output(text):
     send_answer(ANSWER_TEXT, extra=text)
 
+def send_nextnid(unique_id):
+    send_answer(ANSWER_NEXTNID, unique_id)
+
 def current_time():
     """For RPython debug commands: returns the current time."""
     return llop.revdb_get_value(lltype.SignedLongLong, 'c')
@@ -131,7 +138,12 @@
                 cmds = t.revdb_commands
             except AttributeError:
                 cmds = t.revdb_commands = []
-            cmds.append((command_num, func))
+            for old_num, old_func in cmds:
+                if old_num == command_num:
+                    assert old_func is func
+                    break
+            else:
+                cmds.append((command_num, func))
             s_func = self.bookkeeper.immutablevalue(func)
             s_ptr1 = llannotation.SomePtr(ll_ptrtype=_CMDPTR)
             s_str2 = annmodel.SomeString()
@@ -140,3 +152,28 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
+class RegisterAllocationCommand(ExtRegistryEntry):
+    _about_ = register_allocation_command
+
+    def compute_result_annotation(self, s_lambda_func):
+        from rpython.annotator import model as annmodel
+        from rpython.rtyper import llannotation
+
+        lambda_func = s_lambda_func.const
+        t = self.bookkeeper.annotator.translator
+        if t.config.translation.reverse_debugger:
+            func = lambda_func()
+            try:
+                assert t.revdb_allocation_cmd is func
+            except AttributeError:
+                t.revdb_allocation_cmd = func
+            s_func = self.bookkeeper.immutablevalue(func)
+            s_int1 = annmodel.SomeInteger(knowntype=r_longlong)
+            s_ref2 = llannotation.lltype_to_annotation(llmemory.GCREF)
+            self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
+                                             s_func, [s_int1, s_ref2])
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -1,5 +1,5 @@
 import py
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.translator.c.support import cdecl
 from rpython.rlib import exports, revdb
 
@@ -23,25 +23,31 @@
 
 
 def prepare_database(db):
-    FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR,
-                                          lltype.Ptr(rstr.STR)], lltype.Void))
+    FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR, lltype.Ptr(rstr.STR)],
+                                         lltype.Void))
+    ALLOCFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.LONGLONG, llmemory.GCREF],
+                                              lltype.Void))
 
     bk = db.translator.annotator.bookkeeper
     cmds = getattr(db.translator, 'revdb_commands', [])
 
-    array_names = lltype.malloc(rffi.CArray(rffi.INT), len(cmds) + 1,
-                                flavor='raw', immortal=True, zero=True)
-    array_funcs = lltype.malloc(rffi.CArray(FUNCPTR), len(cmds),
-                                flavor='raw', immortal=True, zero=True)
+    S = lltype.Struct('RPY_REVDB_COMMANDS',
+                      ('names', lltype.FixedSizeArray(rffi.INT, len(cmds) + 1)),
+                      ('funcs', lltype.FixedSizeArray(FUNCPTR, len(cmds))),
+                      ('alloc', ALLOCFUNCPTR))
+    s = lltype.malloc(S, flavor='raw', immortal=True, zero=True)
 
     for i, (name, func) in enumerate(cmds):
         fnptr = lltype.getfunctionptr(bk.getdesc(func).getuniquegraph())
         assert lltype.typeOf(fnptr) == FUNCPTR
         assert isinstance(name, int) and name != 0
-        array_names[i] = rffi.cast(rffi.INT, name)
-        array_funcs[i] = fnptr
+        s.names[i] = rffi.cast(rffi.INT, name)
+        s.funcs[i] = fnptr
 
-    exports.EXPORTS_obj2name[array_names._as_obj()] = 'rpy_revdb_command_names'
-    exports.EXPORTS_obj2name[array_funcs._as_obj()] = 'rpy_revdb_command_funcs'
-    db.get(array_names)
-    db.get(array_funcs)
+    allocation_cmd = getattr(db.translator, 'revdb_allocation_cmd', None)
+    if allocation_cmd is not None:
+        s.alloc = lltype.getfunctionptr(
+            bk.getdesc(allocation_cmd).getuniquegraph())
+
+    exports.EXPORTS_obj2name[s._as_obj()] = 'rpy_revdb_commands'
+    db.get(s)
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -4,9 +4,11 @@
 
 # See the corresponding answers for details about messages.
 
-CMD_FORK     = -1      # Message(CMD_FORK)
-CMD_QUIT     = -2      # Message(CMD_QUIT)
-CMD_FORWARD  = -3      # Message(CMD_FORWARD, steps, breakpoint_mode)
+CMD_FORK      = -1     # Message(CMD_FORK)
+CMD_QUIT      = -2     # Message(CMD_QUIT)
+CMD_FORWARD   = -3     # Message(CMD_FORWARD, steps, breakpoint_mode)
+CMD_FUTUREIDS = -4     # Message(CMD_FUTUREIDS, extra=list-of-8bytes-uids)
+CMD_ALLOCATING= -5     # Message(CMD_CREATING, uid, addr)
 # extra commands which are not handled by revdb.c, but
 # by revdb.register_debug_command()
 CMD_PRINT       = 1    # Message(CMD_PRINT, extra=expression)
@@ -50,6 +52,10 @@
 #    Message(ANSWER_MOREINFO, stack_depth)
 ANSWER_MOREINFO   = 21
 
+# sent from CMD_PRINT to record the existence of a recallable object
+#    Message(ANSWER_NEXTNID, unique-id)
+ANSWER_NEXTNID    = 22
+
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -172,6 +172,7 @@
         self.active = child
         self.paused = {1: child.clone()}     # {time: subprocess}
         self.all_breakpoints = AllBreakpoints()
+        self.all_printed_objects = []
 
     def get_current_time(self):
         return self.active.current_time
@@ -313,7 +314,8 @@
         """Print an expression.
         """
         self.active.tainted = True
-        self.active.send(Message(CMD_PRINT, extra=expression))
+        next_nid = len(self.all_printed_objects)
+        self.active.send(Message(CMD_PRINT, next_nid, extra=expression))
         self.active.print_text_answer()
 
     def show_backtrace(self):
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -192,9 +192,10 @@
 
 #define INIT_VERSION_NUMBER   0xd80100
 
-#define CMD_FORK     (-1)
-#define CMD_QUIT     (-2)
-#define CMD_FORWARD  (-3)
+#define CMD_FORK      (-1)
+#define CMD_QUIT      (-2)
+#define CMD_FORWARD   (-3)
+#define CMD_FUTUREIDS (-4)
 
 #define ANSWER_INIT       (-20)
 #define ANSWER_READY      (-21)
@@ -215,6 +216,7 @@
 static uint64_t last_recorded_breakpoint_loc;
 static int last_recorded_breakpoint_num;
 static char breakpoint_mode;
+static uint64_t *future_ids, *future_next_id;
 
 static void attach_gdb(void)
 {
@@ -540,12 +542,33 @@
     }
 }
 
+static void command_future_ids(rpy_revdb_command_t *cmd, char *extra)
+{
+    free(future_ids);
+    if (cmd->extra_size == 0) {
+        future_ids = NULL;
+        rpy_revdb.unique_id_break = 0;
+    }
+    else {
+        assert(cmd->extra_size % sizeof(uint64_t) == 0);
+        future_ids = malloc(cmd->extra_size + sizeof(uint64_t));
+        if (future_ids == NULL) {
+            fprintf(stderr, "out of memory for a buffer of %llu chars\n",
+                    (unsigned long long)cmd->extra_size);
+            exit(1);
+        }
+        memcpy(future_ids, extra, cmd->extra_size);
+        future_ids[cmd->extra_size / sizeof(uint64_t)] = 0;
+    }
+    future_next_id = future_ids;
+}
+
 static void command_default(rpy_revdb_command_t *cmd, char *extra)
 {
     RPyString *s;
     int i;
-    for (i = 0; rpy_revdb_command_names[i] != cmd->cmd; i++) {
-        if (rpy_revdb_command_names[i] == 0) {
+    for (i = 0; rpy_revdb_commands.rp_names[i] != cmd->cmd; i++) {
+        if (rpy_revdb_commands.rp_names[i] == 0) {
             fprintf(stderr, "unknown command %d\n", cmd->cmd);
             exit(1);
         }
@@ -558,16 +581,29 @@
         s = make_rpy_string(cmd->extra_size);
         memcpy(_RPyString_AsString(s), extra, cmd->extra_size);
     }
-    execute_rpy_function(rpy_revdb_command_funcs[i], cmd, s);
+    execute_rpy_function(rpy_revdb_commands.rp_funcs[i], cmd, s);
+}
+
+static void save_state(void)
+{
+    stopped_time = rpy_revdb.stop_point_seen;
+    stopped_uid = rpy_revdb.unique_id_seen;
+    rpy_revdb.unique_id_seen = (-1ULL) << 63;
+}
+
+static void restore_state(void)
+{
+    rpy_revdb.stop_point_seen = stopped_time;
+    rpy_revdb.unique_id_seen = stopped_uid;
+    stopped_time = 0;
+    stopped_uid = 0;
 }
 
 RPY_EXTERN
 void rpy_reverse_db_stop_point(void)
 {
     while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
-        stopped_time = rpy_revdb.stop_point_seen;
-        stopped_uid = rpy_revdb.unique_id_seen;
-        rpy_revdb.unique_id_seen = (-1ULL) << 63;
+        save_state();
         breakpoint_mode = 0;
 
         if (pending_after_forward) {
@@ -599,15 +635,16 @@
                 command_forward(&cmd);
                 break;
 
+            case CMD_FUTUREIDS:
+                command_future_ids(&cmd, extra);
+                break;
+
             default:
                 command_default(&cmd, extra);
                 break;
             }
         }
-        rpy_revdb.stop_point_seen = stopped_time;
-        rpy_revdb.unique_id_seen = stopped_uid;
-        stopped_time = 0;
-        stopped_uid = 0;
+        restore_state();
     }
 }
 
@@ -696,39 +733,21 @@
     }
 }
 
-static void (*unique_id_callback)(void *);
-
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
-    rpy_revdb_t dinfo;
-    rpy_revdb.unique_id_break = 0;
-    disable_io(&dinfo);
-    if (setjmp(jmp_buf_cancel_execution) == 0)
-        unique_id_callback(new_object);
-    enable_io(&dinfo);
+    if (!new_object) {
+        fprintf(stderr, "out of memory: allocation failed, cannot continue\n");
+        exit(1);
+    }
+    if (rpy_revdb_commands.rp_alloc) {
+        save_state();
+        rpy_revdb_commands.rp_alloc(rpy_revdb.unique_id_seen, new_object);
+        restore_state();
+    }
+    rpy_revdb.unique_id_break = *future_next_id++;
     return rpy_revdb.unique_id_seen;
 }
 
-RPY_EXTERN
-void rpy_reverse_db_track_object(long long unique_id, void callback(void *))
-{
-    if (stopped_uid <= 0) {
-        fprintf(stderr, "stopped_uid should not be <= 0\n");
-        return;
-    }
-    if (unique_id <= 0) {
-        fprintf(stderr, "cannot track a prebuilt or debugger-created object\n");
-        return;
-    }
-    if (unique_id < stopped_uid) {
-        fprintf(stderr, "cannot track the creation of an object already created\n");
-        return;
-    }
-    assert(callback != NULL);
-    unique_id_callback = callback;
-    rpy_revdb.unique_id_break = unique_id;
-}
-
 
 /* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -114,8 +114,5 @@
 RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);
-RPY_EXTERN void rpy_reverse_db_track_object(long long unique_id,
-                                            void callback(void *));
-
 
 /* ------------------------------------------------------------ */

From pypy.commits at gmail.com  Tue Jun 21 11:07:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 08:07:00 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: fixed issue in unaligned
 integer store
Message-ID: <57695814.e5acc20a.97705.0bd1@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85307:552b762ff801
Date: 2016-06-21 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/552b762ff801/

Log:	fixed issue in unaligned integer store

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -110,9 +110,8 @@
             self.mc.vperm(Vs, Vs, Vs, Vp)
             self.mc.vsel(Vlo, Vs, Vlo, Vmask)
             self.mc.vsel(Vhi, Vhi, Vs, Vmask)
-            self.mc.stvx(Vlo, indexloc.value, baseloc.value)
-            self.mc.addi(t, baseloc.value, -16)
-            self.mc.stvx(Vhi, indexloc.value, t)
+            self.mc.stvx(Vlo, indexloc.value, t)
+            self.mc.stvx(Vhi, indexloc.value, baseloc.value)
         else:
             if itemsize == 4:
                 self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)

From pypy.commits at gmail.com  Tue Jun 21 11:42:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 08:42:52 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: fix warnings
Message-ID: <5769607c.49c51c0a.f586a.46dd@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85308:5c1da7baf37a
Date: 2016-06-21 03:35 +0100
http://bitbucket.org/pypy/pypy/changeset/5c1da7baf37a/

Log:	fix warnings

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -49,7 +49,7 @@
             ("test_Size_exception", "METH_NOARGS",
              """
                  PyObject* f = PyFloat_FromDouble(1.0);
-                 Py_ssize_t size = PyBytes_Size(f);
+                 PyBytes_Size(f);
 
                  Py_DECREF(f);
                  return NULL;
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -20,8 +20,6 @@
                  """
                  Py_buffer buf;
                  PyObject *str = PyBytes_FromString("hello, world.");
-                 PyObject *result;
-
                  if (PyBuffer_FillInfo(&buf, str, PyBytes_AsString(str), 13,
                                        0, 0)) {
                      return NULL;

From pypy.commits at gmail.com  Tue Jun 21 11:42:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 08:42:54 -0700 (PDT)
Subject: [pypy-commit] pypy testing-cleanup-py3k: Update _sre.c to CPython
 3.3.5 (doesn't compile because we're missing most of the PEP393 API)
Message-ID: <5769607e.2523c20a.e0232.056b@mx.google.com>

Author: Ronan Lamy 
Branch: testing-cleanup-py3k
Changeset: r85309:b60d0188fb25
Date: 2016-06-21 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/b60d0188fb25/

Log:	Update _sre.c to CPython 3.3.5 (doesn't compile because we're
	missing most of the PEP393 API)

diff --git a/pypy/module/cpyext/test/_sre.c b/pypy/module/cpyext/test/_sre.c
--- a/pypy/module/cpyext/test/_sre.c
+++ b/pypy/module/cpyext/test/_sre.c
@@ -164,21 +164,17 @@
 
 /* unicode-specific character predicates */
 
-#if defined(HAVE_UNICODE)
-
-#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch))
-#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch))
-#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch))
-#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch))
-#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_')
+#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL(ch)
+#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE(ch)
+#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK(ch)
+#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM(ch)
+#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM(ch) || (ch) == '_')
 
 static unsigned int sre_lower_unicode(unsigned int ch)
 {
-    return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch));
+    return (unsigned int) Py_UNICODE_TOLOWER(ch);
 }
 
-#endif
-
 LOCAL(int)
 sre_category(SRE_CODE category, unsigned int ch)
 {
@@ -206,7 +202,6 @@
     case SRE_CATEGORY_LOC_NOT_WORD:
         return !SRE_LOC_IS_WORD(ch);
 
-#if defined(HAVE_UNICODE)
     case SRE_CATEGORY_UNI_DIGIT:
         return SRE_UNI_IS_DIGIT(ch);
     case SRE_CATEGORY_UNI_NOT_DIGIT:
@@ -223,24 +218,6 @@
         return SRE_UNI_IS_LINEBREAK(ch);
     case SRE_CATEGORY_UNI_NOT_LINEBREAK:
         return !SRE_UNI_IS_LINEBREAK(ch);
-#else
-    case SRE_CATEGORY_UNI_DIGIT:
-        return SRE_IS_DIGIT(ch);
-    case SRE_CATEGORY_UNI_NOT_DIGIT:
-        return !SRE_IS_DIGIT(ch);
-    case SRE_CATEGORY_UNI_SPACE:
-        return SRE_IS_SPACE(ch);
-    case SRE_CATEGORY_UNI_NOT_SPACE:
-        return !SRE_IS_SPACE(ch);
-    case SRE_CATEGORY_UNI_WORD:
-        return SRE_LOC_IS_WORD(ch);
-    case SRE_CATEGORY_UNI_NOT_WORD:
-        return !SRE_LOC_IS_WORD(ch);
-    case SRE_CATEGORY_UNI_LINEBREAK:
-        return SRE_IS_LINEBREAK(ch);
-    case SRE_CATEGORY_UNI_NOT_LINEBREAK:
-        return !SRE_IS_LINEBREAK(ch);
-#endif
     }
     return 0;
 }
@@ -266,7 +243,7 @@
     if (cursize < minsize) {
         void* stack;
         cursize = minsize+minsize/4+1024;
-        TRACE(("allocate/grow stack %d\n", cursize));
+        TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize));
         stack = PyMem_REALLOC(state->data_stack, cursize);
         if (!stack) {
             data_stack_dealloc(state);
@@ -281,6 +258,7 @@
 /* generate 8-bit version */
 
 #define SRE_CHAR unsigned char
+#define SRE_CHARGET(state, buf, index) ((unsigned char*)buf)[index]
 #define SRE_AT sre_at
 #define SRE_COUNT sre_count
 #define SRE_CHARSET sre_charset
@@ -288,15 +266,11 @@
 #define SRE_MATCH sre_match
 #define SRE_MATCH_CONTEXT sre_match_context
 #define SRE_SEARCH sre_search
-#define SRE_LITERAL_TEMPLATE sre_literal_template
-
-#if defined(HAVE_UNICODE)
 
 #define SRE_RECURSIVE
 #include "_sre.c"
 #undef SRE_RECURSIVE
 
-#undef SRE_LITERAL_TEMPLATE
 #undef SRE_SEARCH
 #undef SRE_MATCH
 #undef SRE_MATCH_CONTEXT
@@ -305,10 +279,15 @@
 #undef SRE_COUNT
 #undef SRE_AT
 #undef SRE_CHAR
-
-/* generate 16-bit unicode version */
-
-#define SRE_CHAR Py_UNICODE
+#undef SRE_CHARGET
+
+/* generate 8/16/32-bit unicode version */
+
+#define SRE_CHAR void
+#define SRE_CHARGET(state, buf, index) \
+    ((state->charsize==1) ? ((Py_UCS1*)buf)[index] : \
+     (state->charsize==2) ? ((Py_UCS2*)buf)[index] : \
+     ((Py_UCS4*)buf)[index])
 #define SRE_AT sre_uat
 #define SRE_COUNT sre_ucount
 #define SRE_CHARSET sre_ucharset
@@ -316,8 +295,6 @@
 #define SRE_MATCH sre_umatch
 #define SRE_MATCH_CONTEXT sre_umatch_context
 #define SRE_SEARCH sre_usearch
-#define SRE_LITERAL_TEMPLATE sre_uliteral_template
-#endif
 
 #endif /* SRE_RECURSIVE */
 
@@ -328,7 +305,7 @@
    settings */
 
 LOCAL(int)
-SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at)
+SRE_AT(SRE_STATE* state, char* ptr, SRE_CODE at)
 {
     /* check if pointer is at given position */
 
@@ -342,16 +319,16 @@
 
     case SRE_AT_BEGINNING_LINE:
         return ((void*) ptr == state->beginning ||
-                SRE_IS_LINEBREAK((int) ptr[-1]));
+                SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, -1)));
 
     case SRE_AT_END:
-        return (((void*) (ptr+1) == state->end &&
-                 SRE_IS_LINEBREAK((int) ptr[0])) ||
+        return (((void*) (ptr+state->charsize) == state->end &&
+                 SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))) ||
                 ((void*) ptr == state->end));
 
     case SRE_AT_END_LINE:
         return ((void*) ptr == state->end ||
-                SRE_IS_LINEBREAK((int) ptr[0]));
+                SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0)));
 
     case SRE_AT_END_STRING:
         return ((void*) ptr == state->end);
@@ -360,57 +337,55 @@
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_IS_WORD((int) ptr[-1]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_IS_WORD((int) ptr[0]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_IS_WORD((int) ptr[-1]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_IS_WORD((int) ptr[0]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
 
     case SRE_AT_LOC_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_LOC_IS_WORD((int) ptr[-1]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_LOC_IS_WORD((int) ptr[0]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_LOC_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_LOC_IS_WORD((int) ptr[-1]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_LOC_IS_WORD((int) ptr[0]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
 
-#if defined(HAVE_UNICODE)
     case SRE_AT_UNI_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_UNI_IS_WORD((int) ptr[-1]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_UNI_IS_WORD((int) ptr[0]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_UNI_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_UNI_IS_WORD((int) ptr[-1]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_UNI_IS_WORD((int) ptr[0]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
-#endif
 
     }
 
@@ -513,28 +488,29 @@
 SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount)
 {
     SRE_CODE chr;
-    SRE_CHAR* ptr = (SRE_CHAR *)state->ptr;
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* ptr = (char *)state->ptr;
+    char* end = (char *)state->end;
     Py_ssize_t i;
 
     /* adjust end */
-    if (maxcount < end - ptr && maxcount != SRE_MAXREPEAT)
-        end = ptr + maxcount;
+    if (maxcount < (end - ptr) / state->charsize && maxcount != SRE_MAXREPEAT)
+        end = ptr + maxcount*state->charsize;
 
     switch (pattern[0]) {
 
     case SRE_OP_IN:
         /* repeated set */
         TRACE(("|%p|%p|COUNT IN\n", pattern, ptr));
-        while (ptr < end && SRE_CHARSET(pattern + 2, *ptr))
-            ptr++;
+        while (ptr < end &&
+               SRE_CHARSET(pattern + 2, SRE_CHARGET(state, ptr, 0)))
+            ptr += state->charsize;
         break;
 
     case SRE_OP_ANY:
         /* repeated dot wildcard. */
         TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr));
-        while (ptr < end && !SRE_IS_LINEBREAK(*ptr))
-            ptr++;
+        while (ptr < end && !SRE_IS_LINEBREAK(SRE_CHARGET(state, ptr, 0)))
+            ptr += state->charsize;
         break;
 
     case SRE_OP_ANY_ALL:
@@ -548,51 +524,52 @@
         /* repeated literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) *ptr == chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) == chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_LITERAL_IGNORE:
         /* repeated literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) == chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_NOT_LITERAL:
         /* repeated non-literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) *ptr != chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_NOT_LITERAL_IGNORE:
         /* repeated non-literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) != chr)
+            ptr += state->charsize;
         break;
 
     default:
         /* repeated single character pattern */
         TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr));
-        while ((SRE_CHAR*) state->ptr < end) {
+        while ((char*) state->ptr < end) {
             i = SRE_MATCH(state, pattern);
             if (i < 0)
                 return i;
             if (!i)
                 break;
         }
-        TRACE(("|%p|%p|COUNT %d\n", pattern, ptr,
-               (SRE_CHAR*) state->ptr - ptr));
-        return (SRE_CHAR*) state->ptr - ptr;
+        TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr,
+               ((char*)state->ptr - ptr)/state->charsize));
+        return ((char*)state->ptr - ptr)/state->charsize;
     }
 
-    TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr));
-    return ptr - (SRE_CHAR*) state->ptr;
+    TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr,
+           (ptr - (char*) state->ptr)/state->charsize));
+    return (ptr - (char*) state->ptr)/state->charsize;
 }
 
 #if 0 /* not used in this release */
@@ -603,19 +580,19 @@
        returns the number of SRE_CODE objects to skip if successful, 0
        if no match */
 
-    SRE_CHAR* end = state->end;
-    SRE_CHAR* ptr = state->ptr;
+    char* end = state->end;
+    char* ptr = state->ptr;
     Py_ssize_t i;
 
     /* check minimal length */
-    if (pattern[3] && (end - ptr) < pattern[3])
+    if (pattern[3] && (end - ptr)/state->charsize < pattern[3])
         return 0;
 
     /* check known prefix */
     if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) {
         /*     */
         for (i = 0; i < pattern[5]; i++)
-            if ((SRE_CODE) ptr[i] != pattern[7 + i])
+            if ((SRE_CODE) SRE_CHARGET(state, ptr, i) != pattern[7 + i])
                 return 0;
         return pattern[0] + 2 * pattern[6];
     }
@@ -678,9 +655,10 @@
 #define DATA_STACK_ALLOC(state, type, ptr) \
 do { \
     alloc_pos = state->data_stack_base; \
-    TRACE(("allocating %s in %d (%d)\n", \
+    TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            SFY(type), alloc_pos, sizeof(type))); \
-    if (state->data_stack_size < alloc_pos+sizeof(type)) { \
+    if (sizeof(type) > state->data_stack_size - alloc_pos) { \
         int j = data_stack_grow(state, sizeof(type)); \
         if (j < 0) return j; \
         if (ctx_pos != -1) \
@@ -692,15 +670,16 @@
 
 #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \
 do { \
-    TRACE(("looking up %s at %d\n", SFY(type), pos)); \
+    TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \
     ptr = (type*)(state->data_stack+pos); \
 } while (0)
 
 #define DATA_STACK_PUSH(state, data, size) \
 do { \
-    TRACE(("copy data in %p to %d (%d)\n", \
+    TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            data, state->data_stack_base, size)); \
-    if (state->data_stack_size < state->data_stack_base+size) { \
+    if (size > state->data_stack_size - state->data_stack_base) { \
         int j = data_stack_grow(state, size); \
         if (j < 0) return j; \
         if (ctx_pos != -1) \
@@ -712,7 +691,8 @@
 
 #define DATA_STACK_POP(state, data, size, discard) \
 do { \
-    TRACE(("copy data to %p from %d (%d)\n", \
+    TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            data, state->data_stack_base-size, size)); \
     memcpy(data, state->data_stack+state->data_stack_base-size, size); \
     if (discard) \
@@ -721,7 +701,8 @@
 
 #define DATA_STACK_POP_DISCARD(state, size) \
 do { \
-    TRACE(("discard data from %d (%d)\n", \
+    TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            state->data_stack_base-size, size)); \
     state->data_stack_base -= size; \
 } while(0)
@@ -784,7 +765,7 @@
 typedef struct {
     Py_ssize_t last_ctx_pos;
     Py_ssize_t jump;
-    SRE_CHAR* ptr;
+    char* ptr;
     SRE_CODE* pattern;
     Py_ssize_t count;
     Py_ssize_t lastmark;
@@ -800,7 +781,7 @@
 LOCAL(Py_ssize_t)
 SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern)
 {
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* end = (char*)state->end;
     Py_ssize_t alloc_pos, ctx_pos = -1;
     Py_ssize_t i, ret = 0;
     Py_ssize_t jump;
@@ -819,14 +800,16 @@
 
 entrance:
 
-    ctx->ptr = (SRE_CHAR *)state->ptr;
+    ctx->ptr = (char *)state->ptr;
 
     if (ctx->pattern[0] == SRE_OP_INFO) {
         /* optimization info block */
         /*  <1=skip> <2=flags> <3=min> ... */
-        if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) {
-            TRACE(("reject (got %d chars, need %d)\n",
-                   (end - ctx->ptr), ctx->pattern[3]));
+        if (ctx->pattern[3] && (end - ctx->ptr)/state->charsize < ctx->pattern[3]) {
+            TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, "
+                   "need %" PY_FORMAT_SIZE_T "d)\n",
+                   (end - ctx->ptr)/state->charsize,
+                   (Py_ssize_t) ctx->pattern[3]));
             RETURN_FAILURE;
         }
         ctx->pattern += ctx->pattern[1] + 1;
@@ -866,10 +849,10 @@
             /*   */
             TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0])
+            if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[0])
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_NOT_LITERAL:
@@ -877,10 +860,10 @@
             /*   */
             TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0])
+            if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) == ctx->pattern[0])
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_SUCCESS:
@@ -903,19 +886,19 @@
             /*   */
             TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0]))
+            if (ctx->ptr >= end || !sre_category(ctx->pattern[0], SRE_CHARGET(state, ctx->ptr, 0)))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_ANY:
             /* match anything (except a newline) */
             /*  */
             TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr));
-            if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0]))
-                RETURN_FAILURE;
-            ctx->ptr++;
+                if (ctx->ptr >= end || SRE_IS_LINEBREAK(SRE_CHARGET(state, ctx->ptr, 0)))
+                    RETURN_FAILURE;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_ANY_ALL:
@@ -924,47 +907,47 @@
             TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr));
             if (ctx->ptr >= end)
                 RETURN_FAILURE;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_IN:
             /* match set member (or non_member) */
             /*    */
             TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr));
-            if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr))
-                RETURN_FAILURE;
+                if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, SRE_CHARGET(state, ctx->ptr, 0)))
+                    RETURN_FAILURE;
             ctx->pattern += ctx->pattern[0];
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_LITERAL_IGNORE:
             TRACE(("|%p|%p|LITERAL_IGNORE %d\n",
                    ctx->pattern, ctx->ptr, ctx->pattern[0]));
             if (ctx->ptr >= end ||
-                state->lower(*ctx->ptr) != state->lower(*ctx->pattern))
+                state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*ctx->pattern))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_NOT_LITERAL_IGNORE:
             TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n",
                    ctx->pattern, ctx->ptr, *ctx->pattern));
             if (ctx->ptr >= end ||
-                state->lower(*ctx->ptr) == state->lower(*ctx->pattern))
+                state->lower(SRE_CHARGET(state, ctx->ptr, 0)) == state->lower(*ctx->pattern))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_IN_IGNORE:
             TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr));
             if (ctx->ptr >= end
                 || !SRE_CHARSET(ctx->pattern+1,
-                                (SRE_CODE)state->lower(*ctx->ptr)))
+                                (SRE_CODE)state->lower(SRE_CHARGET(state, ctx->ptr, 0))))
                 RETURN_FAILURE;
             ctx->pattern += ctx->pattern[0];
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_JUMP:
@@ -987,11 +970,11 @@
             for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) {
                 if (ctx->pattern[1] == SRE_OP_LITERAL &&
                     (ctx->ptr >= end ||
-                     (SRE_CODE) *ctx->ptr != ctx->pattern[2]))
+                     (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[2]))
                     continue;
                 if (ctx->pattern[1] == SRE_OP_IN &&
                     (ctx->ptr >= end ||
-                     !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr)))
+                     !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0))))
                     continue;
                 state->ptr = ctx->ptr;
                 DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1);
@@ -1022,7 +1005,7 @@
             TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr,
                    ctx->pattern[1], ctx->pattern[2]));
 
-            if (ctx->ptr + ctx->pattern[1] > end)
+            if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize)
                 RETURN_FAILURE; /* cannot match */
 
             state->ptr = ctx->ptr;
@@ -1031,7 +1014,7 @@
             RETURN_ON_ERROR(ret);
             DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos);
             ctx->count = ret;
-            ctx->ptr += ctx->count;
+            ctx->ptr += state->charsize * ctx->count;
 
             /* when we arrive here, count contains the number of
                matches, and ctx->ptr points to the tail of the target
@@ -1055,8 +1038,9 @@
                 ctx->u.chr = ctx->pattern[ctx->pattern[0]+1];
                 for (;;) {
                     while (ctx->count >= (Py_ssize_t) ctx->pattern[1] &&
-                           (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) {
-                        ctx->ptr--;
+                           (ctx->ptr >= end ||
+                            SRE_CHARGET(state, ctx->ptr, 0) != ctx->u.chr)) {
+                        ctx->ptr -= state->charsize;
                         ctx->count--;
                     }
                     if (ctx->count < (Py_ssize_t) ctx->pattern[1])
@@ -1071,7 +1055,7 @@
 
                     LASTMARK_RESTORE();
 
-                    ctx->ptr--;
+                    ctx->ptr -= state->charsize;
                     ctx->count--;
                 }
 
@@ -1085,7 +1069,7 @@
                         RETURN_ON_ERROR(ret);
                         RETURN_SUCCESS;
                     }
-                    ctx->ptr--;
+                    ctx->ptr -= state->charsize;
                     ctx->count--;
                     LASTMARK_RESTORE();
                 }
@@ -1105,7 +1089,7 @@
             TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr,
                    ctx->pattern[1], ctx->pattern[2]));
 
-            if (ctx->ptr + ctx->pattern[1] > end)
+            if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize)
                 RETURN_FAILURE; /* cannot match */
 
             state->ptr = ctx->ptr;
@@ -1122,7 +1106,7 @@
                     RETURN_FAILURE;
                 /* advance past minimum matches of repeat */
                 ctx->count = ret;
-                ctx->ptr += ctx->count;
+                ctx->ptr += state->charsize * ctx->count;
             }
 
             if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) {
@@ -1149,7 +1133,7 @@
                     if (ret == 0)
                         break;
                     assert(ret == 1);
-                    ctx->ptr++;
+                    ctx->ptr += state->charsize;
                     ctx->count++;
                     LASTMARK_RESTORE();
                 }
@@ -1201,10 +1185,10 @@
 
             ctx->count = ctx->u.rep->count+1;
 
-            TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern,
+            TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern,
                    ctx->ptr, ctx->count));
 
-            if (ctx->count < ctx->u.rep->pattern[1]) {
+            if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
                 /* not enough matches */
                 ctx->u.rep->count = ctx->count;
                 DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1,
@@ -1218,7 +1202,7 @@
                 RETURN_FAILURE;
             }
 
-            if ((ctx->count < ctx->u.rep->pattern[2] ||
+            if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] ||
                 ctx->u.rep->pattern[2] == SRE_MAXREPEAT) &&
                 state->ptr != ctx->u.rep->last_ptr) {
                 /* we may have enough matches, but if we can
@@ -1264,10 +1248,10 @@
 
             ctx->count = ctx->u.rep->count+1;
 
-            TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern,
+            TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern,
                    ctx->ptr, ctx->count, ctx->u.rep->pattern));
 
-            if (ctx->count < ctx->u.rep->pattern[1]) {
+            if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
                 /* not enough matches */
                 ctx->u.rep->count = ctx->count;
                 DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1,
@@ -1296,7 +1280,7 @@
 
             LASTMARK_RESTORE();
 
-            if ((ctx->count >= ctx->u.rep->pattern[2]
+            if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2]
                 && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) ||
                 state->ptr == ctx->u.rep->last_ptr)
                 RETURN_FAILURE;
@@ -1326,14 +1310,16 @@
                 if (groupref >= state->lastmark) {
                     RETURN_FAILURE;
                 } else {
-                    SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref];
-                    SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1];
+                    char* p = (char*) state->mark[groupref];
+                    char* e = (char*) state->mark[groupref+1];
                     if (!p || !e || e < p)
                         RETURN_FAILURE;
                     while (p < e) {
-                        if (ctx->ptr >= end || *ctx->ptr != *p)
+                        if (ctx->ptr >= end ||
+                            SRE_CHARGET(state, ctx->ptr, 0) != SRE_CHARGET(state, p, 0))
                             RETURN_FAILURE;
-                        p++; ctx->ptr++;
+                        p += state->charsize;
+                        ctx->ptr += state->charsize;
                     }
                 }
             }
@@ -1350,15 +1336,17 @@
                 if (groupref >= state->lastmark) {
                     RETURN_FAILURE;
                 } else {
-                    SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref];
-                    SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1];
+                    char* p = (char*) state->mark[groupref];
+                    char* e = (char*) state->mark[groupref+1];
                     if (!p || !e || e < p)
                         RETURN_FAILURE;
                     while (p < e) {
                         if (ctx->ptr >= end ||
-                            state->lower(*ctx->ptr) != state->lower(*p))
+                            state->lower(SRE_CHARGET(state, ctx->ptr, 0)) !=
+                            state->lower(SRE_CHARGET(state, p, 0)))
                             RETURN_FAILURE;
-                        p++; ctx->ptr++;
+                        p += state->charsize;
+                        ctx->ptr += state->charsize;
                     }
                 }
             }
@@ -1392,7 +1380,7 @@
             /*     */
             TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern,
                    ctx->ptr, ctx->pattern[1]));
-            state->ptr = ctx->ptr - ctx->pattern[1];
+            state->ptr = ctx->ptr - state->charsize * ctx->pattern[1];
             if (state->ptr < state->beginning)
                 RETURN_FAILURE;
             DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2);
@@ -1405,7 +1393,7 @@
             /*     */
             TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern,
                    ctx->ptr, ctx->pattern[1]));
-            state->ptr = ctx->ptr - ctx->pattern[1];
+            state->ptr = ctx->ptr - state->charsize * ctx->pattern[1];
             if (state->ptr >= state->beginning) {
                 DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2);
                 if (ret) {
@@ -1477,7 +1465,8 @@
             TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr));
             goto jump_assert_not;
         case JUMP_NONE:
-            TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret));
+            TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern,
+                   ctx->ptr, ret));
             break;
     }
 
@@ -1487,8 +1476,8 @@
 LOCAL(Py_ssize_t)
 SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
 {
-    SRE_CHAR* ptr = (SRE_CHAR *)state->start;
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* ptr = (char*)state->start;
+    char* end = (char*)state->end;
     Py_ssize_t status = 0;
     Py_ssize_t prefix_len = 0;
     Py_ssize_t prefix_skip = 0;
@@ -1506,9 +1495,9 @@
         if (pattern[3] > 1) {
             /* adjust end point (but make sure we leave at least one
                character in there, so literal search will work) */
-            end -= pattern[3]-1;
+            end -= (pattern[3]-1) * state->charsize;
             if (end <= ptr)
-                end = ptr+1;
+                end = ptr + state->charsize;
         }
 
         if (flags & SRE_INFO_PREFIX) {
@@ -1526,7 +1515,8 @@
         pattern += 1 + pattern[1];
     }
 
-    TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip));
+    TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n",
+           prefix, prefix_len, prefix_skip));
     TRACE(("charset = %p\n", charset));
 
 #if defined(USE_FAST_SEARCH)
@@ -1534,10 +1524,10 @@
         /* pattern starts with a known prefix.  use the overlap
            table to skip forward as fast as we possibly can */
         Py_ssize_t i = 0;
-        end = (SRE_CHAR *)state->end;
+        end = (char *)state->end;
         while (ptr < end) {
             for (;;) {
-                if ((SRE_CODE) ptr[0] != prefix[i]) {
+                if ((SRE_CODE) SRE_CHARGET(state, ptr, 0) != prefix[i]) {
                     if (!i)
                         break;
                     else
@@ -1546,8 +1536,8 @@
                     if (++i == prefix_len) {
                         /* found a potential match */
                         TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr));
-                        state->start = ptr + 1 - prefix_len;
-                        state->ptr = ptr + 1 - prefix_len + prefix_skip;
+                        state->start = ptr - (prefix_len - 1) * state->charsize;
+                        state->ptr = ptr - (prefix_len - prefix_skip - 1) * state->charsize;
                         if (flags & SRE_INFO_LITERAL)
                             return 1; /* we got all of it */
                         status = SRE_MATCH(state, pattern + 2*prefix_skip);
@@ -1559,7 +1549,7 @@
                     break;
                 }
             }
-            ptr++;
+            ptr += state->charsize;
         }
         return 0;
     }
@@ -1569,15 +1559,16 @@
         /* pattern starts with a literal character.  this is used
            for short prefixes, and if fast search is disabled */
         SRE_CODE chr = pattern[1];
-        end = (SRE_CHAR *)state->end;
+        end = (char*)state->end;
         for (;;) {
-            while (ptr < end && (SRE_CODE) ptr[0] != chr)
-                ptr++;
+            while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr)
+                ptr += state->charsize;
             if (ptr >= end)
                 return 0;
             TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr));
             state->start = ptr;
-            state->ptr = ++ptr;
+            ptr += state->charsize;
+            state->ptr = ptr;
             if (flags & SRE_INFO_LITERAL)
                 return 1; /* we got all of it */
             status = SRE_MATCH(state, pattern + 2);
@@ -1586,10 +1577,10 @@
         }
     } else if (charset) {
         /* pattern starts with a character from a known set */
-        end = (SRE_CHAR *)state->end;
+        end = (char*)state->end;
         for (;;) {
-            while (ptr < end && !SRE_CHARSET(charset, ptr[0]))
-                ptr++;
+            while (ptr < end && !SRE_CHARSET(charset, SRE_CHARGET(state, ptr, 0)))
+                ptr += state->charsize;
             if (ptr >= end)
                 return 0;
             TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr));
@@ -1598,13 +1589,14 @@
             status = SRE_MATCH(state, pattern);
             if (status != 0)
                 break;
-            ptr++;
+            ptr += state->charsize;
         }
     } else
         /* general case */
         while (ptr <= end) {
             TRACE(("|%p|%p|SEARCH\n", pattern, ptr));
-            state->start = state->ptr = ptr++;
+            state->start = state->ptr = ptr;
+            ptr += state->charsize;
             status = SRE_MATCH(state, pattern);
             if (status != 0)
                 break;
@@ -1613,25 +1605,32 @@
     return status;
 }
 
-LOCAL(int)
-SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len)
+#if !defined(SRE_RECURSIVE)
+
+/* -------------------------------------------------------------------- */
+/* factories and destructors */
+
+/* see sre.h for object declarations */
+static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int);
+static PyObject*pattern_scanner(PatternObject*, PyObject*, PyObject* kw);
+
+static int
+sre_literal_template(int charsize, char* ptr, Py_ssize_t len)
 {
     /* check if given string is a literal template (i.e. no escapes) */
-    while (len-- > 0)
-        if (*ptr++ == '\\')
+    struct {
+        int charsize;
+    } state = {
+        charsize
+    };
+    while (len-- > 0) {
+        if (SRE_CHARGET((&state), ptr, 0) == '\\')
             return 0;
+        ptr += charsize;
+    }
     return 1;
 }
 
-#if !defined(SRE_RECURSIVE)
-
-/* -------------------------------------------------------------------- */
-/* factories and destructors */
-
-/* see sre.h for object declarations */
-static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int);
-static PyObject*pattern_scanner(PatternObject*, PyObject*);
-
 static PyObject *
 sre_codesize(PyObject* self, PyObject *unused)
 {
@@ -1647,11 +1646,7 @@
     if (flags & SRE_FLAG_LOCALE)
         return Py_BuildValue("i", sre_lower_locale(character));
     if (flags & SRE_FLAG_UNICODE)
-#if defined(HAVE_UNICODE)
         return Py_BuildValue("i", sre_lower_unicode(character));
-#else
-        return Py_BuildValue("i", sre_lower_locale(character));
-#endif
     return Py_BuildValue("i", sre_lower(character));
 }
 
@@ -1670,7 +1665,9 @@
 }
 
 static void*
-getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize, Py_buffer *view)
+getstring(PyObject* string, Py_ssize_t* p_length,
+          int* p_logical_charsize, int* p_charsize,
+          Py_buffer *view)
 {
     /* given a python object, return a data pointer, a length (in
        characters), and a character size.  return NULL if the object
@@ -1684,13 +1681,16 @@
     /* Unicode objects do not support the buffer API. So, get the data
        directly instead. */
     if (PyUnicode_Check(string)) {
-        ptr = (void *)PyUnicode_AS_DATA(string);
-        *p_length = PyUnicode_GET_SIZE(string);
-        *p_charsize = sizeof(Py_UNICODE);
+        if (PyUnicode_READY(string) == -1)
+            return NULL;
+        ptr = PyUnicode_DATA(string);
+        *p_length = PyUnicode_GET_LENGTH(string);
+        *p_charsize = PyUnicode_KIND(string);
+        *p_logical_charsize = 4;
         return ptr;
     }
 
-    /* get pointer to string buffer */
+    /* get pointer to byte string buffer */
     view->len = -1;
     buffer = Py_TYPE(string)->tp_as_buffer;
     if (!buffer || !buffer->bf_getbuffer ||
@@ -1713,10 +1713,6 @@
 
     if (PyBytes_Check(string) || bytes == size)
         charsize = 1;
-#if defined(HAVE_UNICODE)
-    else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE)))
-        charsize = sizeof(Py_UNICODE);
-#endif
     else {
         PyErr_SetString(PyExc_TypeError, "buffer size mismatch");
         goto err;
@@ -1724,6 +1720,7 @@
 
     *p_length = size;
     *p_charsize = charsize;
+    *p_logical_charsize = charsize;
 
     if (ptr == NULL) {
             PyErr_SetString(PyExc_ValueError,
@@ -1744,7 +1741,7 @@
     /* prepare state object */
 
     Py_ssize_t length;
-    int charsize;
+    int logical_charsize, charsize;
     void* ptr;
 
     memset(state, 0, sizeof(SRE_STATE));
@@ -1753,18 +1750,18 @@
     state->lastindex = -1;
 
     state->buffer.buf = NULL;
-    ptr = getstring(string, &length, &charsize, &state->buffer);
+    ptr = getstring(string, &length, &logical_charsize, &charsize, &state->buffer);
     if (!ptr)
         goto err;
 
-    if (charsize == 1 && pattern->charsize > 1) {
+    if (logical_charsize == 1 && pattern->logical_charsize > 1) {
         PyErr_SetString(PyExc_TypeError,
-			"can't use a string pattern on a bytes-like object");
+                        "can't use a string pattern on a bytes-like object");
         goto err;
     }
-    if (charsize > 1 && pattern->charsize == 1) {
+    if (logical_charsize > 1 && pattern->logical_charsize == 1) {
         PyErr_SetString(PyExc_TypeError,
-			"can't use a bytes pattern on a string-like object");
+                        "can't use a bytes pattern on a string-like object");
         goto err;
     }
 
@@ -1779,6 +1776,7 @@
     else if (end > length)
         end = length;
 
+    state->logical_charsize = logical_charsize;
     state->charsize = charsize;
 
     state->beginning = ptr;
@@ -1794,11 +1792,7 @@
     if (pattern->flags & SRE_FLAG_LOCALE)
         state->lower = sre_lower_locale;
     else if (pattern->flags & SRE_FLAG_UNICODE)
-#if defined(HAVE_UNICODE)
         state->lower = sre_lower_unicode;
-#else
-        state->lower = sre_lower_locale;
-#endif
     else
         state->lower = sre_lower;
 
@@ -1905,12 +1899,10 @@
 
     TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr));
 
-    if (state.charsize == 1) {
+    if (state.logical_charsize == 1) {
         status = sre_match(&state, PatternObject_GetCode(self));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_umatch(&state, PatternObject_GetCode(self));
-#endif
     }
 
     TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr));
@@ -1942,12 +1934,10 @@
 
     TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr));
 
-    if (state.charsize == 1) {
+    if (state.logical_charsize == 1) {
         status = sre_search(&state, PatternObject_GetCode(self));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
     }
 
     TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr));
@@ -2089,16 +2079,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2160,13 +2148,13 @@
 
 #if PY_VERSION_HEX >= 0x02020000
 static PyObject*
-pattern_finditer(PatternObject* pattern, PyObject* args)
+pattern_finditer(PatternObject* pattern, PyObject* args, PyObject* kw)
 {
     PyObject* scanner;
     PyObject* search;
     PyObject* iterator;
 
-    scanner = pattern_scanner(pattern, args);
+    scanner = pattern_scanner(pattern, args, kw);
     if (!scanner)
         return NULL;
 
@@ -2219,16 +2207,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2309,7 +2295,7 @@
     int status;
     Py_ssize_t n;
     Py_ssize_t i, b, e;
-    int bint;
+    int logical_charsize, charsize;
     int filter_is_callable;
     Py_buffer view;
 
@@ -2322,16 +2308,10 @@
         /* if not callable, check if it's a literal string */
         int literal;
         view.buf = NULL;
-        ptr = getstring(ptemplate, &n, &bint, &view);
-        b = bint;
+        ptr = getstring(ptemplate, &n, &logical_charsize, &charsize, &view);
+        b = charsize;
         if (ptr) {
-            if (b == 1) {
-		    literal = sre_literal_template((unsigned char *)ptr, n);
-            } else {
-#if defined(HAVE_UNICODE)
-		    literal = sre_uliteral_template((Py_UNICODE *)ptr, n);
-#endif
-            }
+            literal = sre_literal_template(b, ptr, n);
         } else {
             PyErr_Clear();
             literal = 0;
@@ -2375,16 +2355,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2565,35 +2543,35 @@
 }
 
 PyDoc_STRVAR(pattern_match_doc,
-"match(string[, pos[, endpos]]) -> match object or None.\n\n\
+"match(string[, pos[, endpos]]) -> match object or None.\n\
     Matches zero or more characters at the beginning of the string");
 
 PyDoc_STRVAR(pattern_search_doc,
-"search(string[, pos[, endpos]]) -> match object or None.\n\n\
+"search(string[, pos[, endpos]]) -> match object or None.\n\
     Scan through string looking for a match, and return a corresponding\n\
     match object instance. Return None if no position in the string matches.");
 
 PyDoc_STRVAR(pattern_split_doc,
-"split(string[, maxsplit = 0])  -> list.\n\n\
+"split(string[, maxsplit = 0])  -> list.\n\
     Split string by the occurrences of pattern.");
 
 PyDoc_STRVAR(pattern_findall_doc,
-"findall(string[, pos[, endpos]]) -> list.\n\n\
+"findall(string[, pos[, endpos]]) -> list.\n\
    Return a list of all non-overlapping matches of pattern in string.");
 
 PyDoc_STRVAR(pattern_finditer_doc,
-"finditer(string[, pos[, endpos]]) -> iterator.\n\n\
+"finditer(string[, pos[, endpos]]) -> iterator.\n\
     Return an iterator over all non-overlapping matches for the \n\
     RE pattern in string. For each match, the iterator returns a\n\
     match object.");
 
 PyDoc_STRVAR(pattern_sub_doc,
-"sub(repl, string[, count = 0]) -> newstring.\n\n\
+"sub(repl, string[, count = 0]) -> newstring.\n\
     Return the string obtained by replacing the leftmost non-overlapping\n\
     occurrences of pattern in string by the replacement repl.");
 
 PyDoc_STRVAR(pattern_subn_doc,
-"subn(repl, string[, count = 0]) -> (newstring, number of subs)\n\n\
+"subn(repl, string[, count = 0]) -> (newstring, number of subs)\n\
     Return the tuple (new_string, number_of_subs_made) found by replacing\n\
     the leftmost non-overlapping occurrences of pattern with the\n\
     replacement repl.");
@@ -2602,22 +2580,22 @@
 
 static PyMethodDef pattern_methods[] = {
     {"match", (PyCFunction) pattern_match, METH_VARARGS|METH_KEYWORDS,
-	pattern_match_doc},
+        pattern_match_doc},
     {"search", (PyCFunction) pattern_search, METH_VARARGS|METH_KEYWORDS,
-	pattern_search_doc},
+        pattern_search_doc},
     {"sub", (PyCFunction) pattern_sub, METH_VARARGS|METH_KEYWORDS,
-	pattern_sub_doc},
+        pattern_sub_doc},
     {"subn", (PyCFunction) pattern_subn, METH_VARARGS|METH_KEYWORDS,
-	pattern_subn_doc},
+        pattern_subn_doc},
     {"split", (PyCFunction) pattern_split, METH_VARARGS|METH_KEYWORDS,
-	pattern_split_doc},
+        pattern_split_doc},
     {"findall", (PyCFunction) pattern_findall, METH_VARARGS|METH_KEYWORDS,
-	pattern_findall_doc},
+        pattern_findall_doc},
 #if PY_VERSION_HEX >= 0x02020000
-    {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS,
-	pattern_finditer_doc},
+    {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS|METH_KEYWORDS,
+        pattern_finditer_doc},
 #endif
-    {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS},
+    {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS|METH_KEYWORDS},
     {"__copy__", (PyCFunction) pattern_copy, METH_NOARGS},
     {"__deepcopy__", (PyCFunction) pattern_deepcopy, METH_O},
     {NULL, NULL}
@@ -2636,31 +2614,31 @@
     PyVarObject_HEAD_INIT(NULL, 0)
     "_" SRE_MODULE ".SRE_Pattern",
     sizeof(PatternObject), sizeof(SRE_CODE),
-    (destructor)pattern_dealloc,	/* tp_dealloc */
-    0,					/* tp_print */
-    0,					/* tp_getattr */
-    0,					/* tp_setattr */
-    0,					/* tp_reserved */
-    0,					/* tp_repr */
-    0,					/* tp_as_number */
-    0,					/* tp_as_sequence */
-    0,					/* tp_as_mapping */
-    0,					/* tp_hash */
-    0,					/* tp_call */
-    0,					/* tp_str */
-    0,					/* tp_getattro */
-    0,					/* tp_setattro */
-    0,					/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,			/* tp_flags */
-    pattern_doc,			/* tp_doc */
-    0,					/* tp_traverse */
-    0,					/* tp_clear */
-    0,					/* tp_richcompare */
-    offsetof(PatternObject, weakreflist),	/* tp_weaklistoffset */
-    0,					/* tp_iter */
-    0,					/* tp_iternext */
-    pattern_methods,			/* tp_methods */
-    pattern_members,			/* tp_members */
+    (destructor)pattern_dealloc,        /* tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_reserved */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    0,                                  /* tp_getattro */
+    0,                                  /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                 /* tp_flags */
+    pattern_doc,                        /* tp_doc */
+    0,                                  /* tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    offsetof(PatternObject, weakreflist),       /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    pattern_methods,                    /* tp_methods */
+    pattern_members,                    /* tp_members */
 };
 
 static int _validate(PatternObject *self); /* Forward */
@@ -2701,13 +2679,6 @@
     for (i = 0; i < n; i++) {
         PyObject *o = PyList_GET_ITEM(code, i);
         unsigned long value = PyLong_AsUnsignedLong(o);
-        if (value == (unsigned long)-1 && PyErr_Occurred()) {
-            if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-                PyErr_SetString(PyExc_OverflowError,
-                                "regular expression code size limit exceeded");
-            }
-            break;
-        }
         self->code[i] = (SRE_CODE) value;
         if ((unsigned long) self->code[i] != value) {
             PyErr_SetString(PyExc_OverflowError,
@@ -2721,11 +2692,14 @@
         return NULL;
     }
 
-    if (pattern == Py_None)
+    if (pattern == Py_None) {
+        self->logical_charsize = -1;
         self->charsize = -1;
+    }
     else {
         Py_ssize_t p_length;
-        if (!getstring(pattern, &p_length, &self->charsize, &self->view)) {
+        if (!getstring(pattern, &p_length, &self->logical_charsize,
+                       &self->charsize, &self->view)) {
             Py_DECREF(self);
             return NULL;
         }
@@ -2776,8 +2750,7 @@
     \_________\_____/        /
                \____________/
 
-   It also helps that SRE_CODE is always an unsigned type, either 2 bytes or 4
-   bytes wide (the latter if Python is compiled for "wide" unicode support).
+   It also helps that SRE_CODE is always an unsigned type.
 */
 
 /* Defining this one enables tracing of the validator */
@@ -2815,7 +2788,7 @@
         skip = *code;                                   \
         VTRACE(("%lu (skip to %p)\n",                   \
                (unsigned long)skip, code+skip));        \
-        if (code+skip-adj < code || code+skip-adj > end)\
+        if (skip-adj > end-code)                        \
             FAIL;                                       \
         code++;                                         \
     } while (0)
@@ -2848,7 +2821,7 @@
 
         case SRE_OP_CHARSET:
             offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             code += offset;
             break;
@@ -2856,7 +2829,7 @@
         case SRE_OP_BIGCHARSET:
             GET_ARG; /* Number of blocks */
             offset = 256/sizeof(SRE_CODE); /* 256-byte table */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             /* Make sure that each byte points to a valid block */
             for (i = 0; i < 256; i++) {
@@ -2865,7 +2838,7 @@
             }
             code += offset;
             offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             code += offset;
             break;
@@ -2990,13 +2963,13 @@
                     <1=skip> <2=flags> <3=min> <4=max>;
                    If SRE_INFO_PREFIX or SRE_INFO_CHARSET is in the flags,
                    more follows. */
-                SRE_CODE flags, min, max, i;
+                SRE_CODE flags, i;
                 SRE_CODE *newcode;
                 GET_SKIP;
                 newcode = code+skip-1;
                 GET_ARG; flags = arg;
-                GET_ARG; min = arg;
-                GET_ARG; max = arg;
+                GET_ARG;
+                GET_ARG;
                 /* Check that only valid flags are present */
                 if ((flags & ~(SRE_INFO_PREFIX |
                                SRE_INFO_LITERAL |
@@ -3012,15 +2985,15 @@
                     FAIL;
                 /* Validate the prefix */
                 if (flags & SRE_INFO_PREFIX) {
-                    SRE_CODE prefix_len, prefix_skip;
+                    SRE_CODE prefix_len;
                     GET_ARG; prefix_len = arg;
-                    GET_ARG; prefix_skip = arg;
+                    GET_ARG;
                     /* Here comes the prefix string */
-                    if (code+prefix_len < code || code+prefix_len > newcode)
+                    if (prefix_len > newcode-code)
                         FAIL;
                     code += prefix_len;
                     /* And here comes the overlap table */
-                    if (code+prefix_len < code || code+prefix_len > newcode)
+                    if (prefix_len > newcode-code)
                         FAIL;
                     /* Each overlap value should be < prefix_len */
                     for (i = 0; i < prefix_len; i++) {
@@ -3149,7 +3122,7 @@
                to allow arbitrary jumps anywhere in the code; so we just look
                for a JUMP opcode preceding our skip target.
             */
-            if (skip >= 3 && code+skip-3 >= code &&
+            if (skip >= 3 && skip-3 < end-code &&
                 code[skip-3] == SRE_OP_JUMP)
             {
                 VTRACE(("both then and else parts present\n"));
@@ -3261,8 +3234,8 @@
     Py_ssize_t i;
 
     if (index == NULL)
-	/* Default value */
-	return 0;
+        /* Default value */
+        return 0;
 
     if (PyLong_Check(index))
         return PyLong_AsSsize_t(index);
@@ -3590,36 +3563,36 @@
 Match objects always have a boolean value of True.");
 
 PyDoc_STRVAR(match_group_doc,
-"group([group1, ...]) -> str or tuple.\n\n\
+"group([group1, ...]) -> str or tuple.\n\
     Return subgroup(s) of the match by indices or names.\n\
     For 0 returns the entire match.");
 
 PyDoc_STRVAR(match_start_doc,
-"start([group=0]) -> int.\n\n\
+"start([group=0]) -> int.\n\
     Return index of the start of the substring matched by group.");
 
 PyDoc_STRVAR(match_end_doc,
-"end([group=0]) -> int.\n\n\
+"end([group=0]) -> int.\n\
     Return index of the end of the substring matched by group.");
 
 PyDoc_STRVAR(match_span_doc,
-"span([group]) -> tuple.\n\n\
+"span([group]) -> tuple.\n\
     For MatchObject m, return the 2-tuple (m.start(group), m.end(group)).");
 
 PyDoc_STRVAR(match_groups_doc,
-"groups([default=None]) -> tuple.\n\n\
+"groups([default=None]) -> tuple.\n\
     Return a tuple containing all the subgroups of the match, from 1.\n\
     The default argument is used for groups\n\
     that did not participate in the match");
 
 PyDoc_STRVAR(match_groupdict_doc,
-"groupdict([default=None]) -> dict.\n\n\
+"groupdict([default=None]) -> dict.\n\
     Return a dictionary containing all the named subgroups of the match,\n\
     keyed by the subgroup name. The default argument is used for groups\n\
     that did not participate in the match");
 
 PyDoc_STRVAR(match_expand_doc,
-"expand(template) -> str.\n\n\
+"expand(template) -> str.\n\
     Return the string obtained by doing backslash substitution\n\
     on the string template, as done by the sub() method.");
 
@@ -3695,32 +3668,32 @@
     PyVarObject_HEAD_INIT(NULL,0)
     "_" SRE_MODULE ".SRE_Match",
     sizeof(MatchObject), sizeof(Py_ssize_t),
-    (destructor)match_dealloc,	/* tp_dealloc */
-    0,				/* tp_print */
-    0,				/* tp_getattr */
-    0,				/* tp_setattr */
-    0,				/* tp_reserved */
-    0,				/* tp_repr */
-    0,				/* tp_as_number */
-    0,				/* tp_as_sequence */
-    0,				/* tp_as_mapping */
-    0,				/* tp_hash */
-    0,				/* tp_call */
-    0,				/* tp_str */
-    0,				/* tp_getattro */
-    0,				/* tp_setattro */
-    0,				/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,		/* tp_flags */
-    match_doc,			/* tp_doc */
-    0,				/* tp_traverse */
-    0,				/* tp_clear */
-    0,				/* tp_richcompare */
-    0,				/* tp_weaklistoffset */
-    0,				/* tp_iter */
-    0,				/* tp_iternext */
-    match_methods,		/* tp_methods */
-    match_members,		/* tp_members */
-    match_getset,  	        /* tp_getset */
+    (destructor)match_dealloc,  /* tp_dealloc */
+    0,                          /* tp_print */
+    0,                          /* tp_getattr */
+    0,                          /* tp_setattr */
+    0,                          /* tp_reserved */
+    0,                          /* tp_repr */
+    0,                          /* tp_as_number */
+    0,                          /* tp_as_sequence */
+    0,                          /* tp_as_mapping */
+    0,                          /* tp_hash */
+    0,                          /* tp_call */
+    0,                          /* tp_str */
+    0,                          /* tp_getattro */
+    0,                          /* tp_setattro */
+    0,                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,         /* tp_flags */
+    match_doc,                  /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    match_methods,              /* tp_methods */
+    match_members,              /* tp_members */
+    match_getset,               /* tp_getset */
 };
 
 static PyObject*
@@ -3809,12 +3782,10 @@
 
     state->ptr = state->start;
 
-    if (state->charsize == 1) {
+    if (state->logical_charsize == 1) {
         status = sre_match(state, PatternObject_GetCode(self->pattern));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_umatch(state, PatternObject_GetCode(self->pattern));
-#endif
     }
     if (PyErr_Occurred())
         return NULL;
@@ -3842,12 +3813,10 @@
 
     state->ptr = state->start;
 
-    if (state->charsize == 1) {
+    if (state->logical_charsize == 1) {
         status = sre_search(state, PatternObject_GetCode(self->pattern));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_usearch(state, PatternObject_GetCode(self->pattern));
-#endif
     }
     if (PyErr_Occurred())
         return NULL;
@@ -3871,7 +3840,7 @@
 
 #define SCAN_OFF(x) offsetof(ScannerObject, x)
 static PyMemberDef scanner_members[] = {
-    {"pattern",	T_OBJECT,	SCAN_OFF(pattern),	READONLY},
+    {"pattern", T_OBJECT, SCAN_OFF(pattern), READONLY},
     {NULL}  /* Sentinel */
 };
 
@@ -3880,35 +3849,35 @@
     "_" SRE_MODULE ".SRE_Scanner",
     sizeof(ScannerObject), 0,
     (destructor)scanner_dealloc,/* tp_dealloc */
-    0,				/* tp_print */
-    0,				/* tp_getattr */
-    0,				/* tp_setattr */
-    0,				/* tp_reserved */
-    0,				/* tp_repr */
-    0,				/* tp_as_number */
-    0,				/* tp_as_sequence */
-    0,				/* tp_as_mapping */
-    0,				/* tp_hash */
-    0,				/* tp_call */
-    0,				/* tp_str */
-    0,				/* tp_getattro */
-    0,				/* tp_setattro */
-    0,				/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,		/* tp_flags */
-    0,				/* tp_doc */
-    0,				/* tp_traverse */
-    0,				/* tp_clear */
-    0,				/* tp_richcompare */
-    0,				/* tp_weaklistoffset */
-    0,				/* tp_iter */
-    0,				/* tp_iternext */
-    scanner_methods,		/* tp_methods */
-    scanner_members,		/* tp_members */
-    0,				/* tp_getset */
+    0,                          /* tp_print */
+    0,                          /* tp_getattr */
+    0,                          /* tp_setattr */
+    0,                          /* tp_reserved */
+    0,                          /* tp_repr */
+    0,                          /* tp_as_number */
+    0,                          /* tp_as_sequence */
+    0,                          /* tp_as_mapping */
+    0,                          /* tp_hash */
+    0,                          /* tp_call */
+    0,                          /* tp_str */
+    0,                          /* tp_getattro */
+    0,                          /* tp_setattro */
+    0,                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,         /* tp_flags */
+    0,                          /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    scanner_methods,            /* tp_methods */
+    scanner_members,            /* tp_members */
+    0,                          /* tp_getset */
 };
 
 static PyObject*
-pattern_scanner(PatternObject* pattern, PyObject* args)
+pattern_scanner(PatternObject* pattern, PyObject* args, PyObject* kw)
 {
     /* create search state object */
 
@@ -3917,7 +3886,9 @@
     PyObject* string;
     Py_ssize_t start = 0;
     Py_ssize_t end = PY_SSIZE_T_MAX;
-    if (!PyArg_ParseTuple(args, "O|nn:scanner", &string, &start, &end))
+    static char* kwlist[] = { "source", "pos", "endpos", NULL };
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:scanner", kwlist,
+                                     &string, &start, &end))
         return NULL;
 
     /* create scanner object */
@@ -3946,15 +3917,15 @@
 };
 
 static struct PyModuleDef sremodule = {
-	PyModuleDef_HEAD_INIT,
-	"_" SRE_MODULE,
-	NULL,
-	-1,
-	_functions,
-	NULL,
-	NULL,
-	NULL,
-	NULL
+        PyModuleDef_HEAD_INIT,
+        "_" SRE_MODULE,
+        NULL,
+        -1,
+        _functions,
+        NULL,
+        NULL,
+        NULL,
+        NULL
 };
 
 PyMODINIT_FUNC PyInit__sre(void)
@@ -3970,7 +3941,7 @@
 
     m = PyModule_Create(&sremodule);
     if (m == NULL)
-    	return NULL;
+        return NULL;
     d = PyModule_GetDict(m);
 
     x = PyLong_FromLong(SRE_MAGIC);

From pypy.commits at gmail.com  Tue Jun 21 11:55:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 08:55:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge branch testing-cleanup-py3k
Message-ID: <57696360.56311c0a.27d81.4e7d@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85310:df6e701d52a2
Date: 2016-06-21 16:54 +0100
http://bitbucket.org/pypy/pypy/changeset/df6e701d52a2/

Log:	merge branch testing-cleanup-py3k

diff too long, truncating to 2000 out of 2361 lines

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1203,7 +1203,7 @@
 
     def test_const_fold_unicode_subscr(self, monkeypatch):
         source = """def f():
-        return u"abc"[0]
+        return "abc"[0]
         """
         counts = self.count_instructions(source)
         if 0:   # xxx later?
@@ -1211,14 +1211,14 @@
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
-        return u"\U00012345"[0]
+        return "\U00012345"[0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
         source = """def f():
-        return u"\U00012345abcdef"[3]
+        return "\U00012345abcdef"[3]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
@@ -1226,7 +1226,7 @@
 
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
-        return u"\uE01F"[0]
+        return "\uE01F"[0]
         """
         counts = self.count_instructions(source)
         if 0:   # xxx later?
@@ -1236,7 +1236,7 @@
         # getslice is not yet optimized.
         # Still, check a case which yields the empty string.
         source = """def f():
-        return u"abc"[:0]
+        return "abc"[:0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 3, ops.BUILD_SLICE: 1,
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -802,6 +802,21 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
+def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+    from rpython.rlib import rgil
+    # see "Handling of the GIL" above
+    assert cpyext_glob_tid_ptr[0] == 0
+    if pygilstate_release:
+        from pypy.module.cpyext import pystate
+        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
+    else:
+        unlock = gil_release or _gil_auto
+    if unlock:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
+
+
 def make_wrapper_second_level(space, argtypesw, restype,
                               result_kind, error_value, gil):
     from rpython.rlib import rgil
@@ -829,6 +844,7 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
+        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -841,7 +857,6 @@
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
         elif pygilstate_ensure:
-            from pypy.module.cpyext import pystate
             if cpyext_glob_tid_ptr[0] == tid:
                 cpyext_glob_tid_ptr[0] = 0
                 args += (pystate.PyGILState_LOCKED,)
@@ -852,6 +867,10 @@
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
+        if pygilstate_release:
+            gilstate = rffi.cast(lltype.Signed, args[-1])
+        else:
+            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -921,24 +940,13 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
+            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        # see "Handling of the GIL" above
-        assert cpyext_glob_tid_ptr[0] == 0
-        if pygilstate_release:
-            from pypy.module.cpyext import pystate
-            arg = rffi.cast(lltype.Signed, args[-1])
-            unlock = (arg == pystate.PyGILState_UNLOCKED)
-        else:
-            unlock = gil_release or _gil_auto
-        if unlock:
-            rgil.release()
-        else:
-            cpyext_glob_tid_ptr[0] = tid
-
+        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -211,6 +211,7 @@
 PyGILState_STATE = rffi.INT
 PyGILState_LOCKED = 0
 PyGILState_UNLOCKED = 1
+PyGILState_IGNORE = 2
 
 ExecutionContext.cpyext_gilstate_counter_noleave = 0
 
diff --git a/pypy/module/cpyext/test/_sre.c b/pypy/module/cpyext/test/_sre.c
--- a/pypy/module/cpyext/test/_sre.c
+++ b/pypy/module/cpyext/test/_sre.c
@@ -164,21 +164,17 @@
 
 /* unicode-specific character predicates */
 
-#if defined(HAVE_UNICODE)
-
-#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch))
-#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch))
-#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch))
-#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch))
-#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_')
+#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL(ch)
+#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE(ch)
+#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK(ch)
+#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM(ch)
+#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM(ch) || (ch) == '_')
 
 static unsigned int sre_lower_unicode(unsigned int ch)
 {
-    return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch));
+    return (unsigned int) Py_UNICODE_TOLOWER(ch);
 }
 
-#endif
-
 LOCAL(int)
 sre_category(SRE_CODE category, unsigned int ch)
 {
@@ -206,7 +202,6 @@
     case SRE_CATEGORY_LOC_NOT_WORD:
         return !SRE_LOC_IS_WORD(ch);
 
-#if defined(HAVE_UNICODE)
     case SRE_CATEGORY_UNI_DIGIT:
         return SRE_UNI_IS_DIGIT(ch);
     case SRE_CATEGORY_UNI_NOT_DIGIT:
@@ -223,24 +218,6 @@
         return SRE_UNI_IS_LINEBREAK(ch);
     case SRE_CATEGORY_UNI_NOT_LINEBREAK:
         return !SRE_UNI_IS_LINEBREAK(ch);
-#else
-    case SRE_CATEGORY_UNI_DIGIT:
-        return SRE_IS_DIGIT(ch);
-    case SRE_CATEGORY_UNI_NOT_DIGIT:
-        return !SRE_IS_DIGIT(ch);
-    case SRE_CATEGORY_UNI_SPACE:
-        return SRE_IS_SPACE(ch);
-    case SRE_CATEGORY_UNI_NOT_SPACE:
-        return !SRE_IS_SPACE(ch);
-    case SRE_CATEGORY_UNI_WORD:
-        return SRE_LOC_IS_WORD(ch);
-    case SRE_CATEGORY_UNI_NOT_WORD:
-        return !SRE_LOC_IS_WORD(ch);
-    case SRE_CATEGORY_UNI_LINEBREAK:
-        return SRE_IS_LINEBREAK(ch);
-    case SRE_CATEGORY_UNI_NOT_LINEBREAK:
-        return !SRE_IS_LINEBREAK(ch);
-#endif
     }
     return 0;
 }
@@ -266,7 +243,7 @@
     if (cursize < minsize) {
         void* stack;
         cursize = minsize+minsize/4+1024;
-        TRACE(("allocate/grow stack %d\n", cursize));
+        TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize));
         stack = PyMem_REALLOC(state->data_stack, cursize);
         if (!stack) {
             data_stack_dealloc(state);
@@ -281,6 +258,7 @@
 /* generate 8-bit version */
 
 #define SRE_CHAR unsigned char
+#define SRE_CHARGET(state, buf, index) ((unsigned char*)buf)[index]
 #define SRE_AT sre_at
 #define SRE_COUNT sre_count
 #define SRE_CHARSET sre_charset
@@ -288,15 +266,11 @@
 #define SRE_MATCH sre_match
 #define SRE_MATCH_CONTEXT sre_match_context
 #define SRE_SEARCH sre_search
-#define SRE_LITERAL_TEMPLATE sre_literal_template
-
-#if defined(HAVE_UNICODE)
 
 #define SRE_RECURSIVE
 #include "_sre.c"
 #undef SRE_RECURSIVE
 
-#undef SRE_LITERAL_TEMPLATE
 #undef SRE_SEARCH
 #undef SRE_MATCH
 #undef SRE_MATCH_CONTEXT
@@ -305,10 +279,15 @@
 #undef SRE_COUNT
 #undef SRE_AT
 #undef SRE_CHAR
-
-/* generate 16-bit unicode version */
-
-#define SRE_CHAR Py_UNICODE
+#undef SRE_CHARGET
+
+/* generate 8/16/32-bit unicode version */
+
+#define SRE_CHAR void
+#define SRE_CHARGET(state, buf, index) \
+    ((state->charsize==1) ? ((Py_UCS1*)buf)[index] : \
+     (state->charsize==2) ? ((Py_UCS2*)buf)[index] : \
+     ((Py_UCS4*)buf)[index])
 #define SRE_AT sre_uat
 #define SRE_COUNT sre_ucount
 #define SRE_CHARSET sre_ucharset
@@ -316,8 +295,6 @@
 #define SRE_MATCH sre_umatch
 #define SRE_MATCH_CONTEXT sre_umatch_context
 #define SRE_SEARCH sre_usearch
-#define SRE_LITERAL_TEMPLATE sre_uliteral_template
-#endif
 
 #endif /* SRE_RECURSIVE */
 
@@ -328,7 +305,7 @@
    settings */
 
 LOCAL(int)
-SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at)
+SRE_AT(SRE_STATE* state, char* ptr, SRE_CODE at)
 {
     /* check if pointer is at given position */
 
@@ -342,16 +319,16 @@
 
     case SRE_AT_BEGINNING_LINE:
         return ((void*) ptr == state->beginning ||
-                SRE_IS_LINEBREAK((int) ptr[-1]));
+                SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, -1)));
 
     case SRE_AT_END:
-        return (((void*) (ptr+1) == state->end &&
-                 SRE_IS_LINEBREAK((int) ptr[0])) ||
+        return (((void*) (ptr+state->charsize) == state->end &&
+                 SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))) ||
                 ((void*) ptr == state->end));
 
     case SRE_AT_END_LINE:
         return ((void*) ptr == state->end ||
-                SRE_IS_LINEBREAK((int) ptr[0]));
+                SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0)));
 
     case SRE_AT_END_STRING:
         return ((void*) ptr == state->end);
@@ -360,57 +337,55 @@
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_IS_WORD((int) ptr[-1]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_IS_WORD((int) ptr[0]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_IS_WORD((int) ptr[-1]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_IS_WORD((int) ptr[0]) : 0;
+            SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
 
     case SRE_AT_LOC_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_LOC_IS_WORD((int) ptr[-1]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_LOC_IS_WORD((int) ptr[0]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_LOC_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_LOC_IS_WORD((int) ptr[-1]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_LOC_IS_WORD((int) ptr[0]) : 0;
+            SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
 
-#if defined(HAVE_UNICODE)
     case SRE_AT_UNI_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_UNI_IS_WORD((int) ptr[-1]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_UNI_IS_WORD((int) ptr[0]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp != thatp;
 
     case SRE_AT_UNI_NON_BOUNDARY:
         if (state->beginning == state->end)
             return 0;
         thatp = ((void*) ptr > state->beginning) ?
-            SRE_UNI_IS_WORD((int) ptr[-1]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0;
         thisp = ((void*) ptr < state->end) ?
-            SRE_UNI_IS_WORD((int) ptr[0]) : 0;
+            SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0;
         return thisp == thatp;
-#endif
 
     }
 
@@ -513,28 +488,29 @@
 SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount)
 {
     SRE_CODE chr;
-    SRE_CHAR* ptr = (SRE_CHAR *)state->ptr;
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* ptr = (char *)state->ptr;
+    char* end = (char *)state->end;
     Py_ssize_t i;
 
     /* adjust end */
-    if (maxcount < end - ptr && maxcount != SRE_MAXREPEAT)
-        end = ptr + maxcount;
+    if (maxcount < (end - ptr) / state->charsize && maxcount != SRE_MAXREPEAT)
+        end = ptr + maxcount*state->charsize;
 
     switch (pattern[0]) {
 
     case SRE_OP_IN:
         /* repeated set */
         TRACE(("|%p|%p|COUNT IN\n", pattern, ptr));
-        while (ptr < end && SRE_CHARSET(pattern + 2, *ptr))
-            ptr++;
+        while (ptr < end &&
+               SRE_CHARSET(pattern + 2, SRE_CHARGET(state, ptr, 0)))
+            ptr += state->charsize;
         break;
 
     case SRE_OP_ANY:
         /* repeated dot wildcard. */
         TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr));
-        while (ptr < end && !SRE_IS_LINEBREAK(*ptr))
-            ptr++;
+        while (ptr < end && !SRE_IS_LINEBREAK(SRE_CHARGET(state, ptr, 0)))
+            ptr += state->charsize;
         break;
 
     case SRE_OP_ANY_ALL:
@@ -548,51 +524,52 @@
         /* repeated literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) *ptr == chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) == chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_LITERAL_IGNORE:
         /* repeated literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) == chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_NOT_LITERAL:
         /* repeated non-literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) *ptr != chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr)
+            ptr += state->charsize;
         break;
 
     case SRE_OP_NOT_LITERAL_IGNORE:
         /* repeated non-literal */
         chr = pattern[1];
         TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr));
-        while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr)
-            ptr++;
+        while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) != chr)
+            ptr += state->charsize;
         break;
 
     default:
         /* repeated single character pattern */
         TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr));
-        while ((SRE_CHAR*) state->ptr < end) {
+        while ((char*) state->ptr < end) {
             i = SRE_MATCH(state, pattern);
             if (i < 0)
                 return i;
             if (!i)
                 break;
         }
-        TRACE(("|%p|%p|COUNT %d\n", pattern, ptr,
-               (SRE_CHAR*) state->ptr - ptr));
-        return (SRE_CHAR*) state->ptr - ptr;
+        TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr,
+               ((char*)state->ptr - ptr)/state->charsize));
+        return ((char*)state->ptr - ptr)/state->charsize;
     }
 
-    TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr));
-    return ptr - (SRE_CHAR*) state->ptr;
+    TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr,
+           (ptr - (char*) state->ptr)/state->charsize));
+    return (ptr - (char*) state->ptr)/state->charsize;
 }
 
 #if 0 /* not used in this release */
@@ -603,19 +580,19 @@
        returns the number of SRE_CODE objects to skip if successful, 0
        if no match */
 
-    SRE_CHAR* end = state->end;
-    SRE_CHAR* ptr = state->ptr;
+    char* end = state->end;
+    char* ptr = state->ptr;
     Py_ssize_t i;
 
     /* check minimal length */
-    if (pattern[3] && (end - ptr) < pattern[3])
+    if (pattern[3] && (end - ptr)/state->charsize < pattern[3])
         return 0;
 
     /* check known prefix */
     if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) {
         /*     */
         for (i = 0; i < pattern[5]; i++)
-            if ((SRE_CODE) ptr[i] != pattern[7 + i])
+            if ((SRE_CODE) SRE_CHARGET(state, ptr, i) != pattern[7 + i])
                 return 0;
         return pattern[0] + 2 * pattern[6];
     }
@@ -678,9 +655,10 @@
 #define DATA_STACK_ALLOC(state, type, ptr) \
 do { \
     alloc_pos = state->data_stack_base; \
-    TRACE(("allocating %s in %d (%d)\n", \
+    TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            SFY(type), alloc_pos, sizeof(type))); \
-    if (state->data_stack_size < alloc_pos+sizeof(type)) { \
+    if (sizeof(type) > state->data_stack_size - alloc_pos) { \
         int j = data_stack_grow(state, sizeof(type)); \
         if (j < 0) return j; \
         if (ctx_pos != -1) \
@@ -692,15 +670,16 @@
 
 #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \
 do { \
-    TRACE(("looking up %s at %d\n", SFY(type), pos)); \
+    TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \
     ptr = (type*)(state->data_stack+pos); \
 } while (0)
 
 #define DATA_STACK_PUSH(state, data, size) \
 do { \
-    TRACE(("copy data in %p to %d (%d)\n", \
+    TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            data, state->data_stack_base, size)); \
-    if (state->data_stack_size < state->data_stack_base+size) { \
+    if (size > state->data_stack_size - state->data_stack_base) { \
         int j = data_stack_grow(state, size); \
         if (j < 0) return j; \
         if (ctx_pos != -1) \
@@ -712,7 +691,8 @@
 
 #define DATA_STACK_POP(state, data, size, discard) \
 do { \
-    TRACE(("copy data to %p from %d (%d)\n", \
+    TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            data, state->data_stack_base-size, size)); \
     memcpy(data, state->data_stack+state->data_stack_base-size, size); \
     if (discard) \
@@ -721,7 +701,8 @@
 
 #define DATA_STACK_POP_DISCARD(state, size) \
 do { \
-    TRACE(("discard data from %d (%d)\n", \
+    TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \
+           "(%" PY_FORMAT_SIZE_T "d)\n", \
            state->data_stack_base-size, size)); \
     state->data_stack_base -= size; \
 } while(0)
@@ -784,7 +765,7 @@
 typedef struct {
     Py_ssize_t last_ctx_pos;
     Py_ssize_t jump;
-    SRE_CHAR* ptr;
+    char* ptr;
     SRE_CODE* pattern;
     Py_ssize_t count;
     Py_ssize_t lastmark;
@@ -800,7 +781,7 @@
 LOCAL(Py_ssize_t)
 SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern)
 {
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* end = (char*)state->end;
     Py_ssize_t alloc_pos, ctx_pos = -1;
     Py_ssize_t i, ret = 0;
     Py_ssize_t jump;
@@ -819,14 +800,16 @@
 
 entrance:
 
-    ctx->ptr = (SRE_CHAR *)state->ptr;
+    ctx->ptr = (char *)state->ptr;
 
     if (ctx->pattern[0] == SRE_OP_INFO) {
         /* optimization info block */
         /*  <1=skip> <2=flags> <3=min> ... */
-        if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) {
-            TRACE(("reject (got %d chars, need %d)\n",
-                   (end - ctx->ptr), ctx->pattern[3]));
+        if (ctx->pattern[3] && (end - ctx->ptr)/state->charsize < ctx->pattern[3]) {
+            TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, "
+                   "need %" PY_FORMAT_SIZE_T "d)\n",
+                   (end - ctx->ptr)/state->charsize,
+                   (Py_ssize_t) ctx->pattern[3]));
             RETURN_FAILURE;
         }
         ctx->pattern += ctx->pattern[1] + 1;
@@ -866,10 +849,10 @@
             /*   */
             TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0])
+            if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[0])
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_NOT_LITERAL:
@@ -877,10 +860,10 @@
             /*   */
             TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0])
+            if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) == ctx->pattern[0])
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_SUCCESS:
@@ -903,19 +886,19 @@
             /*   */
             TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern,
                    ctx->ptr, *ctx->pattern));
-            if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0]))
+            if (ctx->ptr >= end || !sre_category(ctx->pattern[0], SRE_CHARGET(state, ctx->ptr, 0)))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_ANY:
             /* match anything (except a newline) */
             /*  */
             TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr));
-            if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0]))
-                RETURN_FAILURE;
-            ctx->ptr++;
+                if (ctx->ptr >= end || SRE_IS_LINEBREAK(SRE_CHARGET(state, ctx->ptr, 0)))
+                    RETURN_FAILURE;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_ANY_ALL:
@@ -924,47 +907,47 @@
             TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr));
             if (ctx->ptr >= end)
                 RETURN_FAILURE;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_IN:
             /* match set member (or non_member) */
             /*    */
             TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr));
-            if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr))
-                RETURN_FAILURE;
+                if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, SRE_CHARGET(state, ctx->ptr, 0)))
+                    RETURN_FAILURE;
             ctx->pattern += ctx->pattern[0];
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_LITERAL_IGNORE:
             TRACE(("|%p|%p|LITERAL_IGNORE %d\n",
                    ctx->pattern, ctx->ptr, ctx->pattern[0]));
             if (ctx->ptr >= end ||
-                state->lower(*ctx->ptr) != state->lower(*ctx->pattern))
+                state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*ctx->pattern))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_NOT_LITERAL_IGNORE:
             TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n",
                    ctx->pattern, ctx->ptr, *ctx->pattern));
             if (ctx->ptr >= end ||
-                state->lower(*ctx->ptr) == state->lower(*ctx->pattern))
+                state->lower(SRE_CHARGET(state, ctx->ptr, 0)) == state->lower(*ctx->pattern))
                 RETURN_FAILURE;
             ctx->pattern++;
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_IN_IGNORE:
             TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr));
             if (ctx->ptr >= end
                 || !SRE_CHARSET(ctx->pattern+1,
-                                (SRE_CODE)state->lower(*ctx->ptr)))
+                                (SRE_CODE)state->lower(SRE_CHARGET(state, ctx->ptr, 0))))
                 RETURN_FAILURE;
             ctx->pattern += ctx->pattern[0];
-            ctx->ptr++;
+            ctx->ptr += state->charsize;
             break;
 
         case SRE_OP_JUMP:
@@ -987,11 +970,11 @@
             for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) {
                 if (ctx->pattern[1] == SRE_OP_LITERAL &&
                     (ctx->ptr >= end ||
-                     (SRE_CODE) *ctx->ptr != ctx->pattern[2]))
+                     (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[2]))
                     continue;
                 if (ctx->pattern[1] == SRE_OP_IN &&
                     (ctx->ptr >= end ||
-                     !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr)))
+                     !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0))))
                     continue;
                 state->ptr = ctx->ptr;
                 DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1);
@@ -1022,7 +1005,7 @@
             TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr,
                    ctx->pattern[1], ctx->pattern[2]));
 
-            if (ctx->ptr + ctx->pattern[1] > end)
+            if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize)
                 RETURN_FAILURE; /* cannot match */
 
             state->ptr = ctx->ptr;
@@ -1031,7 +1014,7 @@
             RETURN_ON_ERROR(ret);
             DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos);
             ctx->count = ret;
-            ctx->ptr += ctx->count;
+            ctx->ptr += state->charsize * ctx->count;
 
             /* when we arrive here, count contains the number of
                matches, and ctx->ptr points to the tail of the target
@@ -1055,8 +1038,9 @@
                 ctx->u.chr = ctx->pattern[ctx->pattern[0]+1];
                 for (;;) {
                     while (ctx->count >= (Py_ssize_t) ctx->pattern[1] &&
-                           (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) {
-                        ctx->ptr--;
+                           (ctx->ptr >= end ||
+                            SRE_CHARGET(state, ctx->ptr, 0) != ctx->u.chr)) {
+                        ctx->ptr -= state->charsize;
                         ctx->count--;
                     }
                     if (ctx->count < (Py_ssize_t) ctx->pattern[1])
@@ -1071,7 +1055,7 @@
 
                     LASTMARK_RESTORE();
 
-                    ctx->ptr--;
+                    ctx->ptr -= state->charsize;
                     ctx->count--;
                 }
 
@@ -1085,7 +1069,7 @@
                         RETURN_ON_ERROR(ret);
                         RETURN_SUCCESS;
                     }
-                    ctx->ptr--;
+                    ctx->ptr -= state->charsize;
                     ctx->count--;
                     LASTMARK_RESTORE();
                 }
@@ -1105,7 +1089,7 @@
             TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr,
                    ctx->pattern[1], ctx->pattern[2]));
 
-            if (ctx->ptr + ctx->pattern[1] > end)
+            if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize)
                 RETURN_FAILURE; /* cannot match */
 
             state->ptr = ctx->ptr;
@@ -1122,7 +1106,7 @@
                     RETURN_FAILURE;
                 /* advance past minimum matches of repeat */
                 ctx->count = ret;
-                ctx->ptr += ctx->count;
+                ctx->ptr += state->charsize * ctx->count;
             }
 
             if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) {
@@ -1149,7 +1133,7 @@
                     if (ret == 0)
                         break;
                     assert(ret == 1);
-                    ctx->ptr++;
+                    ctx->ptr += state->charsize;
                     ctx->count++;
                     LASTMARK_RESTORE();
                 }
@@ -1201,10 +1185,10 @@
 
             ctx->count = ctx->u.rep->count+1;
 
-            TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern,
+            TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern,
                    ctx->ptr, ctx->count));
 
-            if (ctx->count < ctx->u.rep->pattern[1]) {
+            if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
                 /* not enough matches */
                 ctx->u.rep->count = ctx->count;
                 DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1,
@@ -1218,7 +1202,7 @@
                 RETURN_FAILURE;
             }
 
-            if ((ctx->count < ctx->u.rep->pattern[2] ||
+            if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] ||
                 ctx->u.rep->pattern[2] == SRE_MAXREPEAT) &&
                 state->ptr != ctx->u.rep->last_ptr) {
                 /* we may have enough matches, but if we can
@@ -1264,10 +1248,10 @@
 
             ctx->count = ctx->u.rep->count+1;
 
-            TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern,
+            TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern,
                    ctx->ptr, ctx->count, ctx->u.rep->pattern));
 
-            if (ctx->count < ctx->u.rep->pattern[1]) {
+            if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
                 /* not enough matches */
                 ctx->u.rep->count = ctx->count;
                 DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1,
@@ -1296,7 +1280,7 @@
 
             LASTMARK_RESTORE();
 
-            if ((ctx->count >= ctx->u.rep->pattern[2]
+            if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2]
                 && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) ||
                 state->ptr == ctx->u.rep->last_ptr)
                 RETURN_FAILURE;
@@ -1326,14 +1310,16 @@
                 if (groupref >= state->lastmark) {
                     RETURN_FAILURE;
                 } else {
-                    SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref];
-                    SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1];
+                    char* p = (char*) state->mark[groupref];
+                    char* e = (char*) state->mark[groupref+1];
                     if (!p || !e || e < p)
                         RETURN_FAILURE;
                     while (p < e) {
-                        if (ctx->ptr >= end || *ctx->ptr != *p)
+                        if (ctx->ptr >= end ||
+                            SRE_CHARGET(state, ctx->ptr, 0) != SRE_CHARGET(state, p, 0))
                             RETURN_FAILURE;
-                        p++; ctx->ptr++;
+                        p += state->charsize;
+                        ctx->ptr += state->charsize;
                     }
                 }
             }
@@ -1350,15 +1336,17 @@
                 if (groupref >= state->lastmark) {
                     RETURN_FAILURE;
                 } else {
-                    SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref];
-                    SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1];
+                    char* p = (char*) state->mark[groupref];
+                    char* e = (char*) state->mark[groupref+1];
                     if (!p || !e || e < p)
                         RETURN_FAILURE;
                     while (p < e) {
                         if (ctx->ptr >= end ||
-                            state->lower(*ctx->ptr) != state->lower(*p))
+                            state->lower(SRE_CHARGET(state, ctx->ptr, 0)) !=
+                            state->lower(SRE_CHARGET(state, p, 0)))
                             RETURN_FAILURE;
-                        p++; ctx->ptr++;
+                        p += state->charsize;
+                        ctx->ptr += state->charsize;
                     }
                 }
             }
@@ -1392,7 +1380,7 @@
             /*     */
             TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern,
                    ctx->ptr, ctx->pattern[1]));
-            state->ptr = ctx->ptr - ctx->pattern[1];
+            state->ptr = ctx->ptr - state->charsize * ctx->pattern[1];
             if (state->ptr < state->beginning)
                 RETURN_FAILURE;
             DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2);
@@ -1405,7 +1393,7 @@
             /*     */
             TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern,
                    ctx->ptr, ctx->pattern[1]));
-            state->ptr = ctx->ptr - ctx->pattern[1];
+            state->ptr = ctx->ptr - state->charsize * ctx->pattern[1];
             if (state->ptr >= state->beginning) {
                 DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2);
                 if (ret) {
@@ -1477,7 +1465,8 @@
             TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr));
             goto jump_assert_not;
         case JUMP_NONE:
-            TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret));
+            TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern,
+                   ctx->ptr, ret));
             break;
     }
 
@@ -1487,8 +1476,8 @@
 LOCAL(Py_ssize_t)
 SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
 {
-    SRE_CHAR* ptr = (SRE_CHAR *)state->start;
-    SRE_CHAR* end = (SRE_CHAR *)state->end;
+    char* ptr = (char*)state->start;
+    char* end = (char*)state->end;
     Py_ssize_t status = 0;
     Py_ssize_t prefix_len = 0;
     Py_ssize_t prefix_skip = 0;
@@ -1506,9 +1495,9 @@
         if (pattern[3] > 1) {
             /* adjust end point (but make sure we leave at least one
                character in there, so literal search will work) */
-            end -= pattern[3]-1;
+            end -= (pattern[3]-1) * state->charsize;
             if (end <= ptr)
-                end = ptr+1;
+                end = ptr + state->charsize;
         }
 
         if (flags & SRE_INFO_PREFIX) {
@@ -1526,7 +1515,8 @@
         pattern += 1 + pattern[1];
     }
 
-    TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip));
+    TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n",
+           prefix, prefix_len, prefix_skip));
     TRACE(("charset = %p\n", charset));
 
 #if defined(USE_FAST_SEARCH)
@@ -1534,10 +1524,10 @@
         /* pattern starts with a known prefix.  use the overlap
            table to skip forward as fast as we possibly can */
         Py_ssize_t i = 0;
-        end = (SRE_CHAR *)state->end;
+        end = (char *)state->end;
         while (ptr < end) {
             for (;;) {
-                if ((SRE_CODE) ptr[0] != prefix[i]) {
+                if ((SRE_CODE) SRE_CHARGET(state, ptr, 0) != prefix[i]) {
                     if (!i)
                         break;
                     else
@@ -1546,8 +1536,8 @@
                     if (++i == prefix_len) {
                         /* found a potential match */
                         TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr));
-                        state->start = ptr + 1 - prefix_len;
-                        state->ptr = ptr + 1 - prefix_len + prefix_skip;
+                        state->start = ptr - (prefix_len - 1) * state->charsize;
+                        state->ptr = ptr - (prefix_len - prefix_skip - 1) * state->charsize;
                         if (flags & SRE_INFO_LITERAL)
                             return 1; /* we got all of it */
                         status = SRE_MATCH(state, pattern + 2*prefix_skip);
@@ -1559,7 +1549,7 @@
                     break;
                 }
             }
-            ptr++;
+            ptr += state->charsize;
         }
         return 0;
     }
@@ -1569,15 +1559,16 @@
         /* pattern starts with a literal character.  this is used
            for short prefixes, and if fast search is disabled */
         SRE_CODE chr = pattern[1];
-        end = (SRE_CHAR *)state->end;
+        end = (char*)state->end;
         for (;;) {
-            while (ptr < end && (SRE_CODE) ptr[0] != chr)
-                ptr++;
+            while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr)
+                ptr += state->charsize;
             if (ptr >= end)
                 return 0;
             TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr));
             state->start = ptr;
-            state->ptr = ++ptr;
+            ptr += state->charsize;
+            state->ptr = ptr;
             if (flags & SRE_INFO_LITERAL)
                 return 1; /* we got all of it */
             status = SRE_MATCH(state, pattern + 2);
@@ -1586,10 +1577,10 @@
         }
     } else if (charset) {
         /* pattern starts with a character from a known set */
-        end = (SRE_CHAR *)state->end;
+        end = (char*)state->end;
         for (;;) {
-            while (ptr < end && !SRE_CHARSET(charset, ptr[0]))
-                ptr++;
+            while (ptr < end && !SRE_CHARSET(charset, SRE_CHARGET(state, ptr, 0)))
+                ptr += state->charsize;
             if (ptr >= end)
                 return 0;
             TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr));
@@ -1598,13 +1589,14 @@
             status = SRE_MATCH(state, pattern);
             if (status != 0)
                 break;
-            ptr++;
+            ptr += state->charsize;
         }
     } else
         /* general case */
         while (ptr <= end) {
             TRACE(("|%p|%p|SEARCH\n", pattern, ptr));
-            state->start = state->ptr = ptr++;
+            state->start = state->ptr = ptr;
+            ptr += state->charsize;
             status = SRE_MATCH(state, pattern);
             if (status != 0)
                 break;
@@ -1613,25 +1605,32 @@
     return status;
 }
 
-LOCAL(int)
-SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len)
+#if !defined(SRE_RECURSIVE)
+
+/* -------------------------------------------------------------------- */
+/* factories and destructors */
+
+/* see sre.h for object declarations */
+static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int);
+static PyObject*pattern_scanner(PatternObject*, PyObject*, PyObject* kw);
+
+static int
+sre_literal_template(int charsize, char* ptr, Py_ssize_t len)
 {
     /* check if given string is a literal template (i.e. no escapes) */
-    while (len-- > 0)
-        if (*ptr++ == '\\')
+    struct {
+        int charsize;
+    } state = {
+        charsize
+    };
+    while (len-- > 0) {
+        if (SRE_CHARGET((&state), ptr, 0) == '\\')
             return 0;
+        ptr += charsize;
+    }
     return 1;
 }
 
-#if !defined(SRE_RECURSIVE)
-
-/* -------------------------------------------------------------------- */
-/* factories and destructors */
-
-/* see sre.h for object declarations */
-static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int);
-static PyObject*pattern_scanner(PatternObject*, PyObject*);
-
 static PyObject *
 sre_codesize(PyObject* self, PyObject *unused)
 {
@@ -1647,11 +1646,7 @@
     if (flags & SRE_FLAG_LOCALE)
         return Py_BuildValue("i", sre_lower_locale(character));
     if (flags & SRE_FLAG_UNICODE)
-#if defined(HAVE_UNICODE)
         return Py_BuildValue("i", sre_lower_unicode(character));
-#else
-        return Py_BuildValue("i", sre_lower_locale(character));
-#endif
     return Py_BuildValue("i", sre_lower(character));
 }
 
@@ -1670,7 +1665,9 @@
 }
 
 static void*
-getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize, Py_buffer *view)
+getstring(PyObject* string, Py_ssize_t* p_length,
+          int* p_logical_charsize, int* p_charsize,
+          Py_buffer *view)
 {
     /* given a python object, return a data pointer, a length (in
        characters), and a character size.  return NULL if the object
@@ -1684,13 +1681,16 @@
     /* Unicode objects do not support the buffer API. So, get the data
        directly instead. */
     if (PyUnicode_Check(string)) {
-        ptr = (void *)PyUnicode_AS_DATA(string);
-        *p_length = PyUnicode_GET_SIZE(string);
-        *p_charsize = sizeof(Py_UNICODE);
+        if (PyUnicode_READY(string) == -1)
+            return NULL;
+        ptr = PyUnicode_DATA(string);
+        *p_length = PyUnicode_GET_LENGTH(string);
+        *p_charsize = PyUnicode_KIND(string);
+        *p_logical_charsize = 4;
         return ptr;
     }
 
-    /* get pointer to string buffer */
+    /* get pointer to byte string buffer */
     view->len = -1;
     buffer = Py_TYPE(string)->tp_as_buffer;
     if (!buffer || !buffer->bf_getbuffer ||
@@ -1713,10 +1713,6 @@
 
     if (PyBytes_Check(string) || bytes == size)
         charsize = 1;
-#if defined(HAVE_UNICODE)
-    else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE)))
-        charsize = sizeof(Py_UNICODE);
-#endif
     else {
         PyErr_SetString(PyExc_TypeError, "buffer size mismatch");
         goto err;
@@ -1724,6 +1720,7 @@
 
     *p_length = size;
     *p_charsize = charsize;
+    *p_logical_charsize = charsize;
 
     if (ptr == NULL) {
             PyErr_SetString(PyExc_ValueError,
@@ -1744,7 +1741,7 @@
     /* prepare state object */
 
     Py_ssize_t length;
-    int charsize;
+    int logical_charsize, charsize;
     void* ptr;
 
     memset(state, 0, sizeof(SRE_STATE));
@@ -1753,18 +1750,18 @@
     state->lastindex = -1;
 
     state->buffer.buf = NULL;
-    ptr = getstring(string, &length, &charsize, &state->buffer);
+    ptr = getstring(string, &length, &logical_charsize, &charsize, &state->buffer);
     if (!ptr)
         goto err;
 
-    if (charsize == 1 && pattern->charsize > 1) {
+    if (logical_charsize == 1 && pattern->logical_charsize > 1) {
         PyErr_SetString(PyExc_TypeError,
-			"can't use a string pattern on a bytes-like object");
+                        "can't use a string pattern on a bytes-like object");
         goto err;
     }
-    if (charsize > 1 && pattern->charsize == 1) {
+    if (logical_charsize > 1 && pattern->logical_charsize == 1) {
         PyErr_SetString(PyExc_TypeError,
-			"can't use a bytes pattern on a string-like object");
+                        "can't use a bytes pattern on a string-like object");
         goto err;
     }
 
@@ -1779,6 +1776,7 @@
     else if (end > length)
         end = length;
 
+    state->logical_charsize = logical_charsize;
     state->charsize = charsize;
 
     state->beginning = ptr;
@@ -1794,11 +1792,7 @@
     if (pattern->flags & SRE_FLAG_LOCALE)
         state->lower = sre_lower_locale;
     else if (pattern->flags & SRE_FLAG_UNICODE)
-#if defined(HAVE_UNICODE)
         state->lower = sre_lower_unicode;
-#else
-        state->lower = sre_lower_locale;
-#endif
     else
         state->lower = sre_lower;
 
@@ -1905,12 +1899,10 @@
 
     TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr));
 
-    if (state.charsize == 1) {
+    if (state.logical_charsize == 1) {
         status = sre_match(&state, PatternObject_GetCode(self));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_umatch(&state, PatternObject_GetCode(self));
-#endif
     }
 
     TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr));
@@ -1942,12 +1934,10 @@
 
     TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr));
 
-    if (state.charsize == 1) {
+    if (state.logical_charsize == 1) {
         status = sre_search(&state, PatternObject_GetCode(self));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
     }
 
     TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr));
@@ -2089,16 +2079,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2160,13 +2148,13 @@
 
 #if PY_VERSION_HEX >= 0x02020000
 static PyObject*
-pattern_finditer(PatternObject* pattern, PyObject* args)
+pattern_finditer(PatternObject* pattern, PyObject* args, PyObject* kw)
 {
     PyObject* scanner;
     PyObject* search;
     PyObject* iterator;
 
-    scanner = pattern_scanner(pattern, args);
+    scanner = pattern_scanner(pattern, args, kw);
     if (!scanner)
         return NULL;
 
@@ -2219,16 +2207,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2309,7 +2295,7 @@
     int status;
     Py_ssize_t n;
     Py_ssize_t i, b, e;
-    int bint;
+    int logical_charsize, charsize;
     int filter_is_callable;
     Py_buffer view;
 
@@ -2322,16 +2308,10 @@
         /* if not callable, check if it's a literal string */
         int literal;
         view.buf = NULL;
-        ptr = getstring(ptemplate, &n, &bint, &view);
-        b = bint;
+        ptr = getstring(ptemplate, &n, &logical_charsize, &charsize, &view);
+        b = charsize;
         if (ptr) {
-            if (b == 1) {
-		    literal = sre_literal_template((unsigned char *)ptr, n);
-            } else {
-#if defined(HAVE_UNICODE)
-		    literal = sre_uliteral_template((Py_UNICODE *)ptr, n);
-#endif
-            }
+            literal = sre_literal_template(b, ptr, n);
         } else {
             PyErr_Clear();
             literal = 0;
@@ -2375,16 +2355,14 @@
 
         state.ptr = state.start;
 
-        if (state.charsize == 1) {
+        if (state.logical_charsize == 1) {
             status = sre_search(&state, PatternObject_GetCode(self));
         } else {
-#if defined(HAVE_UNICODE)
             status = sre_usearch(&state, PatternObject_GetCode(self));
-#endif
         }
 
-	if (PyErr_Occurred())
-	    goto error;
+        if (PyErr_Occurred())
+            goto error;
 
         if (status <= 0) {
             if (status == 0)
@@ -2565,35 +2543,35 @@
 }
 
 PyDoc_STRVAR(pattern_match_doc,
-"match(string[, pos[, endpos]]) -> match object or None.\n\n\
+"match(string[, pos[, endpos]]) -> match object or None.\n\
     Matches zero or more characters at the beginning of the string");
 
 PyDoc_STRVAR(pattern_search_doc,
-"search(string[, pos[, endpos]]) -> match object or None.\n\n\
+"search(string[, pos[, endpos]]) -> match object or None.\n\
     Scan through string looking for a match, and return a corresponding\n\
     match object instance. Return None if no position in the string matches.");
 
 PyDoc_STRVAR(pattern_split_doc,
-"split(string[, maxsplit = 0])  -> list.\n\n\
+"split(string[, maxsplit = 0])  -> list.\n\
     Split string by the occurrences of pattern.");
 
 PyDoc_STRVAR(pattern_findall_doc,
-"findall(string[, pos[, endpos]]) -> list.\n\n\
+"findall(string[, pos[, endpos]]) -> list.\n\
    Return a list of all non-overlapping matches of pattern in string.");
 
 PyDoc_STRVAR(pattern_finditer_doc,
-"finditer(string[, pos[, endpos]]) -> iterator.\n\n\
+"finditer(string[, pos[, endpos]]) -> iterator.\n\
     Return an iterator over all non-overlapping matches for the \n\
     RE pattern in string. For each match, the iterator returns a\n\
     match object.");
 
 PyDoc_STRVAR(pattern_sub_doc,
-"sub(repl, string[, count = 0]) -> newstring.\n\n\
+"sub(repl, string[, count = 0]) -> newstring.\n\
     Return the string obtained by replacing the leftmost non-overlapping\n\
     occurrences of pattern in string by the replacement repl.");
 
 PyDoc_STRVAR(pattern_subn_doc,
-"subn(repl, string[, count = 0]) -> (newstring, number of subs)\n\n\
+"subn(repl, string[, count = 0]) -> (newstring, number of subs)\n\
     Return the tuple (new_string, number_of_subs_made) found by replacing\n\
     the leftmost non-overlapping occurrences of pattern with the\n\
     replacement repl.");
@@ -2602,22 +2580,22 @@
 
 static PyMethodDef pattern_methods[] = {
     {"match", (PyCFunction) pattern_match, METH_VARARGS|METH_KEYWORDS,
-	pattern_match_doc},
+        pattern_match_doc},
     {"search", (PyCFunction) pattern_search, METH_VARARGS|METH_KEYWORDS,
-	pattern_search_doc},
+        pattern_search_doc},
     {"sub", (PyCFunction) pattern_sub, METH_VARARGS|METH_KEYWORDS,
-	pattern_sub_doc},
+        pattern_sub_doc},
     {"subn", (PyCFunction) pattern_subn, METH_VARARGS|METH_KEYWORDS,
-	pattern_subn_doc},
+        pattern_subn_doc},
     {"split", (PyCFunction) pattern_split, METH_VARARGS|METH_KEYWORDS,
-	pattern_split_doc},
+        pattern_split_doc},
     {"findall", (PyCFunction) pattern_findall, METH_VARARGS|METH_KEYWORDS,
-	pattern_findall_doc},
+        pattern_findall_doc},
 #if PY_VERSION_HEX >= 0x02020000
-    {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS,
-	pattern_finditer_doc},
+    {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS|METH_KEYWORDS,
+        pattern_finditer_doc},
 #endif
-    {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS},
+    {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS|METH_KEYWORDS},
     {"__copy__", (PyCFunction) pattern_copy, METH_NOARGS},
     {"__deepcopy__", (PyCFunction) pattern_deepcopy, METH_O},
     {NULL, NULL}
@@ -2636,31 +2614,31 @@
     PyVarObject_HEAD_INIT(NULL, 0)
     "_" SRE_MODULE ".SRE_Pattern",
     sizeof(PatternObject), sizeof(SRE_CODE),
-    (destructor)pattern_dealloc,	/* tp_dealloc */
-    0,					/* tp_print */
-    0,					/* tp_getattr */
-    0,					/* tp_setattr */
-    0,					/* tp_reserved */
-    0,					/* tp_repr */
-    0,					/* tp_as_number */
-    0,					/* tp_as_sequence */
-    0,					/* tp_as_mapping */
-    0,					/* tp_hash */
-    0,					/* tp_call */
-    0,					/* tp_str */
-    0,					/* tp_getattro */
-    0,					/* tp_setattro */
-    0,					/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,			/* tp_flags */
-    pattern_doc,			/* tp_doc */
-    0,					/* tp_traverse */
-    0,					/* tp_clear */
-    0,					/* tp_richcompare */
-    offsetof(PatternObject, weakreflist),	/* tp_weaklistoffset */
-    0,					/* tp_iter */
-    0,					/* tp_iternext */
-    pattern_methods,			/* tp_methods */
-    pattern_members,			/* tp_members */
+    (destructor)pattern_dealloc,        /* tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_reserved */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    0,                                  /* tp_getattro */
+    0,                                  /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                 /* tp_flags */
+    pattern_doc,                        /* tp_doc */
+    0,                                  /* tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    offsetof(PatternObject, weakreflist),       /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    pattern_methods,                    /* tp_methods */
+    pattern_members,                    /* tp_members */
 };
 
 static int _validate(PatternObject *self); /* Forward */
@@ -2701,13 +2679,6 @@
     for (i = 0; i < n; i++) {
         PyObject *o = PyList_GET_ITEM(code, i);
         unsigned long value = PyLong_AsUnsignedLong(o);
-        if (value == (unsigned long)-1 && PyErr_Occurred()) {
-            if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-                PyErr_SetString(PyExc_OverflowError,
-                                "regular expression code size limit exceeded");
-            }
-            break;
-        }
         self->code[i] = (SRE_CODE) value;
         if ((unsigned long) self->code[i] != value) {
             PyErr_SetString(PyExc_OverflowError,
@@ -2721,11 +2692,14 @@
         return NULL;
     }
 
-    if (pattern == Py_None)
+    if (pattern == Py_None) {
+        self->logical_charsize = -1;
         self->charsize = -1;
+    }
     else {
         Py_ssize_t p_length;
-        if (!getstring(pattern, &p_length, &self->charsize, &self->view)) {
+        if (!getstring(pattern, &p_length, &self->logical_charsize,
+                       &self->charsize, &self->view)) {
             Py_DECREF(self);
             return NULL;
         }
@@ -2776,8 +2750,7 @@
     \_________\_____/        /
                \____________/
 
-   It also helps that SRE_CODE is always an unsigned type, either 2 bytes or 4
-   bytes wide (the latter if Python is compiled for "wide" unicode support).
+   It also helps that SRE_CODE is always an unsigned type.
 */
 
 /* Defining this one enables tracing of the validator */
@@ -2815,7 +2788,7 @@
         skip = *code;                                   \
         VTRACE(("%lu (skip to %p)\n",                   \
                (unsigned long)skip, code+skip));        \
-        if (code+skip-adj < code || code+skip-adj > end)\
+        if (skip-adj > end-code)                        \
             FAIL;                                       \
         code++;                                         \
     } while (0)
@@ -2848,7 +2821,7 @@
 
         case SRE_OP_CHARSET:
             offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             code += offset;
             break;
@@ -2856,7 +2829,7 @@
         case SRE_OP_BIGCHARSET:
             GET_ARG; /* Number of blocks */
             offset = 256/sizeof(SRE_CODE); /* 256-byte table */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             /* Make sure that each byte points to a valid block */
             for (i = 0; i < 256; i++) {
@@ -2865,7 +2838,7 @@
             }
             code += offset;
             offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */
-            if (code+offset < code || code+offset > end)
+            if (offset > end-code)
                 FAIL;
             code += offset;
             break;
@@ -2990,13 +2963,13 @@
                     <1=skip> <2=flags> <3=min> <4=max>;
                    If SRE_INFO_PREFIX or SRE_INFO_CHARSET is in the flags,
                    more follows. */
-                SRE_CODE flags, min, max, i;
+                SRE_CODE flags, i;
                 SRE_CODE *newcode;
                 GET_SKIP;
                 newcode = code+skip-1;
                 GET_ARG; flags = arg;
-                GET_ARG; min = arg;
-                GET_ARG; max = arg;
+                GET_ARG;
+                GET_ARG;
                 /* Check that only valid flags are present */
                 if ((flags & ~(SRE_INFO_PREFIX |
                                SRE_INFO_LITERAL |
@@ -3012,15 +2985,15 @@
                     FAIL;
                 /* Validate the prefix */
                 if (flags & SRE_INFO_PREFIX) {
-                    SRE_CODE prefix_len, prefix_skip;
+                    SRE_CODE prefix_len;
                     GET_ARG; prefix_len = arg;
-                    GET_ARG; prefix_skip = arg;
+                    GET_ARG;
                     /* Here comes the prefix string */
-                    if (code+prefix_len < code || code+prefix_len > newcode)
+                    if (prefix_len > newcode-code)
                         FAIL;
                     code += prefix_len;
                     /* And here comes the overlap table */
-                    if (code+prefix_len < code || code+prefix_len > newcode)
+                    if (prefix_len > newcode-code)
                         FAIL;
                     /* Each overlap value should be < prefix_len */
                     for (i = 0; i < prefix_len; i++) {
@@ -3149,7 +3122,7 @@
                to allow arbitrary jumps anywhere in the code; so we just look
                for a JUMP opcode preceding our skip target.
             */
-            if (skip >= 3 && code+skip-3 >= code &&
+            if (skip >= 3 && skip-3 < end-code &&
                 code[skip-3] == SRE_OP_JUMP)
             {
                 VTRACE(("both then and else parts present\n"));
@@ -3261,8 +3234,8 @@
     Py_ssize_t i;
 
     if (index == NULL)
-	/* Default value */
-	return 0;
+        /* Default value */
+        return 0;
 
     if (PyLong_Check(index))
         return PyLong_AsSsize_t(index);
@@ -3590,36 +3563,36 @@
 Match objects always have a boolean value of True.");
 
 PyDoc_STRVAR(match_group_doc,
-"group([group1, ...]) -> str or tuple.\n\n\
+"group([group1, ...]) -> str or tuple.\n\
     Return subgroup(s) of the match by indices or names.\n\
     For 0 returns the entire match.");
 
 PyDoc_STRVAR(match_start_doc,
-"start([group=0]) -> int.\n\n\
+"start([group=0]) -> int.\n\
     Return index of the start of the substring matched by group.");
 
 PyDoc_STRVAR(match_end_doc,
-"end([group=0]) -> int.\n\n\
+"end([group=0]) -> int.\n\
     Return index of the end of the substring matched by group.");
 
 PyDoc_STRVAR(match_span_doc,
-"span([group]) -> tuple.\n\n\
+"span([group]) -> tuple.\n\
     For MatchObject m, return the 2-tuple (m.start(group), m.end(group)).");
 
 PyDoc_STRVAR(match_groups_doc,
-"groups([default=None]) -> tuple.\n\n\
+"groups([default=None]) -> tuple.\n\
     Return a tuple containing all the subgroups of the match, from 1.\n\
     The default argument is used for groups\n\
     that did not participate in the match");
 
 PyDoc_STRVAR(match_groupdict_doc,
-"groupdict([default=None]) -> dict.\n\n\
+"groupdict([default=None]) -> dict.\n\
     Return a dictionary containing all the named subgroups of the match,\n\
     keyed by the subgroup name. The default argument is used for groups\n\
     that did not participate in the match");
 
 PyDoc_STRVAR(match_expand_doc,
-"expand(template) -> str.\n\n\
+"expand(template) -> str.\n\
     Return the string obtained by doing backslash substitution\n\
     on the string template, as done by the sub() method.");
 
@@ -3695,32 +3668,32 @@
     PyVarObject_HEAD_INIT(NULL,0)
     "_" SRE_MODULE ".SRE_Match",
     sizeof(MatchObject), sizeof(Py_ssize_t),
-    (destructor)match_dealloc,	/* tp_dealloc */
-    0,				/* tp_print */
-    0,				/* tp_getattr */
-    0,				/* tp_setattr */
-    0,				/* tp_reserved */
-    0,				/* tp_repr */
-    0,				/* tp_as_number */
-    0,				/* tp_as_sequence */
-    0,				/* tp_as_mapping */
-    0,				/* tp_hash */
-    0,				/* tp_call */
-    0,				/* tp_str */
-    0,				/* tp_getattro */
-    0,				/* tp_setattro */
-    0,				/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,		/* tp_flags */
-    match_doc,			/* tp_doc */
-    0,				/* tp_traverse */
-    0,				/* tp_clear */
-    0,				/* tp_richcompare */
-    0,				/* tp_weaklistoffset */
-    0,				/* tp_iter */
-    0,				/* tp_iternext */
-    match_methods,		/* tp_methods */
-    match_members,		/* tp_members */
-    match_getset,  	        /* tp_getset */
+    (destructor)match_dealloc,  /* tp_dealloc */
+    0,                          /* tp_print */
+    0,                          /* tp_getattr */
+    0,                          /* tp_setattr */
+    0,                          /* tp_reserved */
+    0,                          /* tp_repr */
+    0,                          /* tp_as_number */
+    0,                          /* tp_as_sequence */
+    0,                          /* tp_as_mapping */
+    0,                          /* tp_hash */
+    0,                          /* tp_call */
+    0,                          /* tp_str */
+    0,                          /* tp_getattro */
+    0,                          /* tp_setattro */
+    0,                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,         /* tp_flags */
+    match_doc,                  /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    match_methods,              /* tp_methods */
+    match_members,              /* tp_members */
+    match_getset,               /* tp_getset */
 };
 
 static PyObject*
@@ -3809,12 +3782,10 @@
 
     state->ptr = state->start;
 
-    if (state->charsize == 1) {
+    if (state->logical_charsize == 1) {
         status = sre_match(state, PatternObject_GetCode(self->pattern));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_umatch(state, PatternObject_GetCode(self->pattern));
-#endif
     }
     if (PyErr_Occurred())
         return NULL;
@@ -3842,12 +3813,10 @@
 
     state->ptr = state->start;
 
-    if (state->charsize == 1) {
+    if (state->logical_charsize == 1) {
         status = sre_search(state, PatternObject_GetCode(self->pattern));
     } else {
-#if defined(HAVE_UNICODE)
         status = sre_usearch(state, PatternObject_GetCode(self->pattern));
-#endif
     }
     if (PyErr_Occurred())
         return NULL;
@@ -3871,7 +3840,7 @@
 
 #define SCAN_OFF(x) offsetof(ScannerObject, x)
 static PyMemberDef scanner_members[] = {
-    {"pattern",	T_OBJECT,	SCAN_OFF(pattern),	READONLY},
+    {"pattern", T_OBJECT, SCAN_OFF(pattern), READONLY},
     {NULL}  /* Sentinel */
 };
 
@@ -3880,35 +3849,35 @@
     "_" SRE_MODULE ".SRE_Scanner",
     sizeof(ScannerObject), 0,
     (destructor)scanner_dealloc,/* tp_dealloc */
-    0,				/* tp_print */
-    0,				/* tp_getattr */
-    0,				/* tp_setattr */
-    0,				/* tp_reserved */
-    0,				/* tp_repr */
-    0,				/* tp_as_number */
-    0,				/* tp_as_sequence */
-    0,				/* tp_as_mapping */
-    0,				/* tp_hash */
-    0,				/* tp_call */
-    0,				/* tp_str */
-    0,				/* tp_getattro */
-    0,				/* tp_setattro */
-    0,				/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,		/* tp_flags */
-    0,				/* tp_doc */
-    0,				/* tp_traverse */
-    0,				/* tp_clear */
-    0,				/* tp_richcompare */
-    0,				/* tp_weaklistoffset */
-    0,				/* tp_iter */
-    0,				/* tp_iternext */
-    scanner_methods,		/* tp_methods */
-    scanner_members,		/* tp_members */
-    0,				/* tp_getset */
+    0,                          /* tp_print */
+    0,                          /* tp_getattr */
+    0,                          /* tp_setattr */
+    0,                          /* tp_reserved */
+    0,                          /* tp_repr */
+    0,                          /* tp_as_number */
+    0,                          /* tp_as_sequence */
+    0,                          /* tp_as_mapping */
+    0,                          /* tp_hash */
+    0,                          /* tp_call */
+    0,                          /* tp_str */
+    0,                          /* tp_getattro */
+    0,                          /* tp_setattro */
+    0,                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,         /* tp_flags */
+    0,                          /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    scanner_methods,            /* tp_methods */
+    scanner_members,            /* tp_members */
+    0,                          /* tp_getset */
 };
 
 static PyObject*
-pattern_scanner(PatternObject* pattern, PyObject* args)
+pattern_scanner(PatternObject* pattern, PyObject* args, PyObject* kw)
 {
     /* create search state object */
 
@@ -3917,7 +3886,9 @@
     PyObject* string;
     Py_ssize_t start = 0;
     Py_ssize_t end = PY_SSIZE_T_MAX;
-    if (!PyArg_ParseTuple(args, "O|nn:scanner", &string, &start, &end))
+    static char* kwlist[] = { "source", "pos", "endpos", NULL };
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:scanner", kwlist,
+                                     &string, &start, &end))
         return NULL;
 
     /* create scanner object */
@@ -3946,15 +3917,15 @@
 };
 
 static struct PyModuleDef sremodule = {
-	PyModuleDef_HEAD_INIT,
-	"_" SRE_MODULE,
-	NULL,
-	-1,
-	_functions,
-	NULL,
-	NULL,
-	NULL,
-	NULL
+        PyModuleDef_HEAD_INIT,
+        "_" SRE_MODULE,
+        NULL,
+        -1,
+        _functions,
+        NULL,
+        NULL,
+        NULL,
+        NULL
 };
 
 PyMODINIT_FUNC PyInit__sre(void)
@@ -3970,7 +3941,7 @@
 
     m = PyModule_Create(&sremodule);
     if (m == NULL)
-    	return NULL;
+        return NULL;
     d = PyModule_GetDict(m);
 
     x = PyLong_FromLong(SRE_MAGIC);
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import os
 import pytest
 
 def pytest_configure(config):
@@ -23,3 +23,14 @@
 def pytest_funcarg__api(request):
     return request.cls.api
 
+if os.name == 'nt':
+    @pytest.yield_fixture(autouse=True, scope='session')
+    def prevent_dialog_box():
+        """Do not open dreaded dialog box on segfault on Windows"""
+        import ctypes
+        SEM_NOGPFAULTERRORBOX = 0x0002  # From MSDN
+        old_err_mode = ctypes.windll.kernel32.GetErrorMode()
+        new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX
+        ctypes.windll.kernel32.SetErrorMode(new_err_mode)
+        yield
+        ctypes.windll.kernel32.SetErrorMode(old_err_mode)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -48,15 +48,6 @@
 
 /* foo methods */
 
-static void
-foo_dealloc(fooobject *foop)
-{
-    PyObject_Del(foop);
-}
-
-
-/* foo methods-as-attributes */
-
 static PyObject *
 foo_copy(fooobject *self)
 {
@@ -200,7 +191,7 @@
     sizeof(fooobject),       /*tp_size*/
     0,                       /*tp_itemsize*/
     /* methods */
-    (destructor)foo_dealloc, /*tp_dealloc*/
+    0,                       /*tp_dealloc*/
     0,                       /*tp_print*/
     0,                       /*tp_getattr*/
     0,                       /*tp_setattr*/
@@ -468,9 +459,9 @@
 PyTypeObject InitErrType = {
     PyObject_HEAD_INIT(NULL)
     0,
-    "foo.InitErr",
-    sizeof(PyObject),
-    0,
+    "foo.InitErrType",
+    sizeof(PyObject),/*tp_basicsize*/
+    0,          /*tp_itemsize*/
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
     0,          /*tp_getattr*/
@@ -513,12 +504,12 @@
     0,          /*tp_dictoffset*/
 
     initerrtype_init,          /*tp_init*/
-    0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
+    0,          /*tp_alloc*/
     0,          /*tp_new*/
-    0,          /*tp_free  Low-level free-memory routine */
-    0,          /*tp_is_gc For PyObject_IS_GC */
+    0,          /*tp_free*/
+    0,          /*tp_is_gc*/
     0,          /*tp_bases*/
-    0,          /*tp_mro method resolution order */
+    0,          /*tp_mro*/
     0,          /*tp_cache*/
     0,          /*tp_subclasses*/
     0           /*tp_weaklist*/
diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/support.py
@@ -0,0 +1,68 @@
+import os
+import py
+from sys import platform
+
+if os.name != 'nt':
+    so_ext = 'so'
+else:
+    so_ext = 'dll'
+
+def c_compile(cfilenames, outputfilename,
+        compile_extra=None, link_extra=None,
+        include_dirs=None, libraries=None, library_dirs=None):
+    compile_extra = compile_extra or []
+    link_extra = link_extra or []
+    include_dirs = include_dirs or []
+    libraries = libraries or []
+    library_dirs = library_dirs or []
+    if platform == 'win32':
+        link_extra = link_extra + ['/DEBUG'] # generate .pdb file
+    if platform == 'darwin':
+        # support Fink & Darwinports
+        for s in ('/sw/', '/opt/local/'):
+            if (s + 'include' not in include_dirs
+                    and os.path.exists(s + 'include')):
+                include_dirs.append(s + 'include')
+            if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
+                library_dirs.append(s + 'lib')
+
+    outputfilename = py.path.local(outputfilename).new(ext=so_ext)
+    saved_environ = os.environ.copy()
+    try:
+        _build(
+            cfilenames, outputfilename,
+            compile_extra, link_extra,
+            include_dirs, libraries, library_dirs)
+    finally:
+        # workaround for a distutils bugs where some env vars can
+        # become longer and longer every time it is used
+        for key, value in saved_environ.items():
+            if os.environ.get(key) != value:
+                os.environ[key] = value
+    return outputfilename
+
+def _build(cfilenames, outputfilename, compile_extra, link_extra,
+        include_dirs, libraries, library_dirs):
+    from distutils.ccompiler import new_compiler
+    from distutils import sysconfig
+    compiler = new_compiler(force=1)
+    sysconfig.customize_compiler(compiler) # XXX
+    objects = []
+    for cfile in cfilenames:
+        cfile = py.path.local(cfile)
+        old = cfile.dirpath().chdir()
+        try:
+            res = compiler.compile([cfile.basename],
+                include_dirs=include_dirs, extra_preargs=compile_extra)
+            assert len(res) == 1
+            cobjfile = py.path.local(res[0])
+            assert cobjfile.check()
+            objects.append(str(cobjfile))
+        finally:
+            old.chdir()
+
+    compiler.link_shared_object(
+        objects, str(outputfilename),
+        libraries=libraries,
+        extra_preargs=link_extra,
+        library_dirs=library_dirs)
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -31,7 +31,7 @@
                  if(s->ob_type->tp_basicsize != expected_size)
                  {
                      printf("tp_basicsize==%ld\\n",
-                            (long)s->ob_type->tp_basicsize); 
+                            (long)s->ob_type->tp_basicsize);
                      result = 0;
                  }
                  Py_DECREF(s);
@@ -53,7 +53,6 @@
              """
                  PyObject *s, *t;
                  char* c;
-                 Py_ssize_t len;
 
                  s = PyByteArray_FromStringAndSize(NULL, 4);
                  if (s == NULL)
@@ -84,7 +83,6 @@
             ("mutable", "METH_NOARGS",

From pypy.commits at gmail.com  Tue Jun 21 12:31:01 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 09:31:01 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: (ppc) implemented vec_int_sub
Message-ID: <57696bc5.56311c0a.27d81.5d24@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85312:03a28e77e69d
Date: 2016-06-21 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/03a28e77e69d/

Log:	(ppc) implemented vec_int_sub

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -624,6 +624,13 @@
     vadduhm = VX(4, XO8=64)
     vaddubm = VX(4, XO8=0)
 
+    vsubudm = VX(4, XO8=1216)
+    vsubuwm = VX(4, XO8=1152)
+    vsubuhm = VX(4, XO8=1088)
+    vsububm = VX(4, XO8=1024)
+
+
+
     # shift, perm and select
     lvsl = XV(31, XO1=6)
     lvsr = XV(31, XO1=38)
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -131,6 +131,21 @@
         elif size == 8:
             self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
 
+    def emit_vec_int_sub(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsububm(resloc.value, loc0.value, loc1.value)
+        elif size == 2:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuhm(resloc.value, loc0.value, loc1.value)
+        elif size == 4:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuwm(resloc.value, loc0.value, loc1.value)
+        elif size == 8:
+            self.mc.vsubudm(resloc.value, loc0.value, loc1.value)
+
     def emit_vec_float_add(self, op, arglocs, resloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value

From pypy.commits at gmail.com  Tue Jun 21 12:31:03 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 09:31:03 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: extend the integer vector test
Message-ID: <57696bc7.4f8d1c0a.7cbba.6153@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85313:277106ec536b
Date: 2016-06-21 18:29 +0200
http://bitbucket.org/pypy/pypy/changeset/277106ec536b/

Log:	extend the integer vector test

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -178,6 +178,29 @@
         elif itemsize == 8:
             self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
 
+    def emit_vec_int_mul(self, op, arglocs, resloc):
+        loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 1:
+            self.mc.PMULLW(loc0, loc1)
+        elif itemsize == 2:
+            self.mc.PMULLW(loc0, loc1)
+        elif itemsize == 4:
+            self.mc.PMULLD(loc0, loc1)
+        else:
+            # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
+            # There is no 64x64 bit packed mul. For 8 bit either. It is questionable if it gives any benefit?
+            not_implemented("int8/64 mul")
+
+    def emit_vec_int_and(self, op, arglocs, resloc):
+        self.mc.PAND(resloc, arglocs[0])
+
+    def emit_vec_int_or(self, op, arglocs, resloc):
+        self.mc.POR(resloc, arglocs[0])
+
+    def emit_vec_int_xor(self, op, arglocs, resloc):
+        self.mc.PXOR(resloc, arglocs[0])
+
     #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc):
     #    self.implement_guard(guard_token)
 
@@ -293,47 +316,6 @@
     #    # entries before) become ones
     #    self.mc.PCMPEQ(loc, temp, sizeloc.value)
 
-    #def genop_vec_int_mul(self, op, arglocs, resloc):
-    #    loc0, loc1, itemsize_loc = arglocs
-    #    itemsize = itemsize_loc.value
-    #    if itemsize == 2:
-    #        self.mc.PMULLW(loc0, loc1)
-    #    elif itemsize == 4:
-    #        self.mc.PMULLD(loc0, loc1)
-    #    else:
-    #        # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
-    #        # There is no 64x64 bit packed mul. For 8 bit either. It is questionable if it gives any benefit?
-    #        not_implemented("int8/64 mul")
-
-    #def genop_vec_int_sub(self, op, arglocs, resloc):
-    #    loc0, loc1, size_loc = arglocs
-    #    size = size_loc.value
-    #    if size == 1:
-    #        self.mc.PSUBB(loc0, loc1)
-    #    elif size == 2:
-    #        self.mc.PSUBW(loc0, loc1)
-    #    elif size == 4:
-    #        self.mc.PSUBD(loc0, loc1)
-    #    elif size == 8:
-    #        self.mc.PSUBQ(loc0, loc1)
-
-    #def genop_vec_int_and(self, op, arglocs, resloc):
-    #    self.mc.PAND(resloc, arglocs[0])
-
-    #def genop_vec_int_or(self, op, arglocs, resloc):
-    #    self.mc.POR(resloc, arglocs[0])
-
-    #def genop_vec_int_xor(self, op, arglocs, resloc):
-    #    self.mc.PXOR(resloc, arglocs[0])
-
-    #def genop_vec_float_truediv(self, op, arglocs, resloc):
-    #    loc0, loc1, sizeloc = arglocs
-    #    size = sizeloc.value
-    #    if size == 4:
-    #        self.mc.DIVPS(loc0, loc1)
-    #    elif size == 8:
-    #        self.mc.DIVPD(loc0, loc1)
-
     #def genop_vec_float_abs(self, op, arglocs, resloc):
     #    src, sizeloc = arglocs
     #    size = sizeloc.value
@@ -648,12 +630,17 @@
         return [resloc, loc0, loc1, imm(size)]
 
     prepare_vec_int_add = prepare_vec_arith
-    #prepare_vec_int_sub = prepare_vec_arith
-    #prepare_vec_int_mul = prepare_vec_arith
+    prepare_vec_int_sub = prepare_vec_arith
+    prepare_vec_int_mul = prepare_vec_arith
     prepare_vec_float_add = prepare_vec_arith
     prepare_vec_float_sub = prepare_vec_arith
     prepare_vec_float_mul = prepare_vec_arith
     prepare_vec_float_truediv = prepare_vec_arith
+
+    # logic functions
+    prepare_vec_int_and = prepare_vec_arith
+    prepare_vec_int_or = prepare_vec_arith
+    prepare_vec_int_xor = prepare_vec_arith
     del prepare_vec_arith
 
     def _prepare_vec_store(self, op):
@@ -691,13 +678,6 @@
     #prepare_vec_float_abs = prepare_vec_arith_unary
     #del prepare_vec_arith_unary
 
-    #def prepare_vec_logic(self, op):
-    #    lhs = op.getarg(0)
-    #    assert isinstance(lhs, VectorOp)
-    #    args = op.getarglist()
-    #    source = self.make_sure_var_in_reg(op.getarg(1), args)
-    #    result = self.xrm.force_result_in_reg(op, op.getarg(0), args)
-    #    self.perform(op, [source, imm(lhs.bytesize)], result)
 
     #def prepare_vec_float_eq(self, op):
     #    assert isinstance(op, VectorOp)
@@ -712,11 +692,6 @@
     #prepare_vec_int_eq = prepare_vec_float_eq
     #prepare_vec_int_ne = prepare_vec_float_eq
 
-    #prepare_vec_int_and = prepare_vec_logic
-    #prepare_vec_int_or = prepare_vec_logic
-    #prepare_vec_int_xor = prepare_vec_logic
-    #del prepare_vec_logic
-
     #def prepare_vec_pack_i(self, op):
     #    # new_res = vec_pack_i(res, src, index, count)
     #    assert isinstance(op, VectorOp)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -52,10 +52,11 @@
             array = self.arrays.pop()
             free_raw_storage(array)
 
- at pytest.fixture(scope='session')
+ at pytest.fixture(scope='function')
 def rawstorage(request):
     rs = RawStorage()
     request.addfinalizer(rs.clear)
+    request.cls.a
     return rs
 
 integers_64bit = st.integers(min_value=-2**63, max_value=2**63-1)
@@ -70,7 +71,7 @@
             return rfloat.NAN
         return rfloat.copysign(rfloat.INFINITY, v1 * v2)
 
-class VectorizeTests:
+class VectorizeTests(object):
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
     def setup_method(self, method):
@@ -126,10 +127,20 @@
         rawstorage.clear()
 
     #@given(st.data())
-    def test_vector_simple_int(self):
+    @pytest.mark.parametrize('func,type', [
+        (lambda a,b: intmask(a+b), rffi.SIGNED),
+        (lambda a,b: intmask(a+b), rffi.UNSIGNED),
+        (lambda a,b: intmask(a+b), rffi.INT),
+        (lambda a,b: intmask(a+b), rffi.UINT),
+        (lambda a,b: intmask(a+b), rffi.SHORT),
+        (lambda a,b: intmask(a+b), rffi.USHORT),
+        (lambda a,b: intmask(a+b), rffi.CHAR),
+        (lambda a,b: intmask(a+b), rffi.UCHAR),
+    ])
+    def test_vector_simple_int(self, func, type):
+        func = always_inline(func)
 
-        type = rffi.SIGNED
-        size = rffi.sizeof(rffi.SIGNED)
+        size = rffi.sizeof(type)
         myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
         def f(bytecount, va, vb, vc):
             i = 0
@@ -137,25 +148,25 @@
                 myjitdriver.jit_merge_point()
                 a = raw_storage_getitem(type,va,i)
                 b = raw_storage_getitem(type,vb,i)
-                c = a+b
+                c = func(a,b)
                 raw_storage_setitem(vc, i, rffi.cast(type,c))
                 i += size
 
-        rawstorage = RawStorage()
         #la = data.draw(st.lists(integers_64bit, min_size=10, max_size=150))
         la = [1] * 10
         l = len(la)
         #lb = data.draw(st.lists(integers_64bit, min_size=l, max_size=l))
         lb = [0] * 10
 
-        va = rawstorage.new(la, lltype.Signed)
-        vb = rawstorage.new(lb, lltype.Signed)
-        vc = rawstorage.new(None, lltype.Signed, size=l)
+        rawstorage = RawStorage()
+        va = rawstorage.new(la, type)
+        vb = rawstorage.new(lb, type)
+        vc = rawstorage.new(None, type, size=l)
         self.meta_interp(f, [l*size, va, vb, vc])
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
-            assert intmask(la[i] + lb[i]) == c
+            assert func(la[i], lb[i]) == c
 
         rawstorage.clear()
 

From pypy.commits at gmail.com  Tue Jun 21 12:30:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Jun 2016 09:30:59 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: (ppc) implemented vec_int_add
Message-ID: <57696bc3.8aa6c20a.15839.ffffeb27@mx.google.com>

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85311:771785582074
Date: 2016-06-21 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/771785582074/

Log:	(ppc) implemented vec_int_add

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -620,6 +620,9 @@
 
     # arith & logic
     vaddudm = VX(4, XO8=192)
+    vadduwm = VX(4, XO8=128)
+    vadduhm = VX(4, XO8=64)
+    vaddubm = VX(4, XO8=0)
 
     # shift, perm and select
     lvsl = XV(31, XO1=6)
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -123,11 +123,11 @@
         resloc, loc0, loc1, size_loc = arglocs
         size = size_loc.value
         if size == 1:
-            raise NotImplementedError
+            self.mc.vaddubm(resloc.value, loc0.value, loc1.value)
         elif size == 2:
-            raise NotImplementedError
+            self.mc.vadduhm(resloc.value, loc0.value, loc1.value)
         elif size == 4:
-            raise NotImplementedError
+            self.mc.vadduwm(resloc.value, loc0.value, loc1.value)
         elif size == 8:
             self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
 

From pypy.commits at gmail.com  Tue Jun 21 12:44:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Jun 2016 09:44:06 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <57696ed6.50991c0a.088e.691f@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85314:a9996f5e2819
Date: 2016-06-21 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/a9996f5e2819/

Log:	in-progress

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -15,6 +15,7 @@
 CMD_LOCALS      = 3
 CMD_BREAKPOINTS = 4
 CMD_MOREINFO    = 5
+CMD_ATTACHID    = 6
 ANSWER_TEXT     = 20
 ANSWER_MOREINFO = 21
 ANSWER_NEXTNID  = 22
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -756,8 +756,8 @@
     def OP_DEBUG_PRINT(self, op):
         # XXX
         from rpython.rtyper.lltypesystem.rstr import STR
-        format = []
-        argv = []
+        format = ['{%d} ']
+        argv = ['(int)getpid()']
         free_line = ""
         for arg in op.args:
             T = arg.concretetype
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -7,6 +7,7 @@
 from rpython.translator.revdb.process import Breakpoint
 
 r_cmdline = re.compile(r"(\S+)\s*(.*)")
+r_dollar_num = re.compile(r"\$(\d+)\b")
 
 
 class RevDebugControl(object):
@@ -193,7 +194,9 @@
 
     def command_print(self, argument):
         """Print an expression"""
-        self.pgroup.print_cmd(argument)
+        # locate which $NUM appear used in the expression
+        nids = map(int, r_dollar_num.findall(argument))
+        self.pgroup.print_cmd(argument, nids=nids)
     command_p = command_print
 
     def command_backtrace(self, argument):
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
--- a/rpython/translator/revdb/message.py
+++ b/rpython/translator/revdb/message.py
@@ -8,7 +8,6 @@
 CMD_QUIT      = -2     # Message(CMD_QUIT)
 CMD_FORWARD   = -3     # Message(CMD_FORWARD, steps, breakpoint_mode)
 CMD_FUTUREIDS = -4     # Message(CMD_FUTUREIDS, extra=list-of-8bytes-uids)
-CMD_ALLOCATING= -5     # Message(CMD_CREATING, uid, addr)
 # extra commands which are not handled by revdb.c, but
 # by revdb.register_debug_command()
 CMD_PRINT       = 1    # Message(CMD_PRINT, extra=expression)
@@ -17,6 +16,7 @@
 CMD_BREAKPOINTS = 4    # Message(CMD_BREAKPOINTS, stack_depth,
                        #         extra="\0-separated names")
 CMD_MOREINFO    = 5    # Message(CMD_MOREINFO)
+CMD_ATTACHID    = 6    # Message(CMD_ATTACHID, small-num, unique-id)
 
 
 # the first message sent by the first child:
@@ -73,7 +73,13 @@
         self.extra = extra
 
     def __repr__(self):
-        return 'Message(%d, %d, %d, %d, %r)' % (self.cmd, self.arg1,
+        cmd = self.cmd
+        for key, value in globals().items():
+            if (key.startswith('CMD_') or key.startswith('ANSWER_')) and (
+                    value == cmd):
+                cmd = key
+                break
+        return 'Message(%s, %d, %d, %d, %r)' % (cmd, self.arg1,
                                                 self.arg2, self.arg3,
                                                 self.extra)
 
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -47,11 +47,18 @@
     It can be either the one started with --revdb-replay, or a fork.
     """
 
-    def __init__(self, pid, control_socket, breakpoints_cache=AllBreakpoints()):
+    def __init__(self, pid, control_socket,
+                 breakpoints_cache=AllBreakpoints(),
+                 printed_objects=frozenset()):
         self.pid = pid
         self.control_socket = control_socket
         self.tainted = False
         self.breakpoints_cache = breakpoints_cache    # don't mutate this
+        self.printed_objects = printed_objects        # don't mutate this
+        # ^^^ frozenset containing the uids of the objects that are
+        #     either already discovered in this child
+        #     (if uid < currently_created_objects), or that will
+        #     automatically be discovered when we move forward
 
     def _recv_all(self, size):
         pieces = []
@@ -64,6 +71,7 @@
         return ''.join(pieces)
 
     def send(self, msg):
+        print 'SENT:', self.pid, msg
         binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
                              msg.arg1, msg.arg2, msg.arg3)
         self.control_socket.sendall(binary + msg.extra)
@@ -72,7 +80,9 @@
         binary = self._recv_all(struct.calcsize("iIqqq"))
         cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
         extra = self._recv_all(size)
-        return Message(cmd, arg1, arg2, arg3, extra)
+        msg = Message(cmd, arg1, arg2, arg3, extra)
+        print 'RECV:', self.pid, msg
+        return msg
 
     def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
         msg = self.recv()
@@ -107,7 +117,8 @@
         child_pid = msg.arg1
         self.expect_ready()
         other = ReplayProcess(child_pid, s1,
-                              breakpoints_cache=self.breakpoints_cache)
+                              breakpoints_cache=self.breakpoints_cache,
+                              printed_objects=self.printed_objects)
         other.expect_ready()
         return other
 
@@ -138,7 +149,7 @@
         self.update_times(msg)
         return bkpt
 
-    def print_text_answer(self):
+    def print_text_answer(self, pgroup=None):
         while True:
             msg = self.recv()
             if msg.cmd == ANSWER_TEXT:
@@ -147,6 +158,16 @@
             elif msg.cmd == ANSWER_READY:
                 self.update_times(msg)
                 break
+            elif msg.cmd == ANSWER_NEXTNID and pgroup is not None:
+                uid = msg.arg1
+                if uid < pgroup.initial_uid:
+                    continue     # created before the first stop point, ignore
+                self.printed_objects = self.printed_objects.union([uid])
+                new_nid = len(pgroup.all_printed_objects_lst)
+                nid = pgroup.all_printed_objects.setdefault(uid, new_nid)
+                if nid == new_nid:
+                    pgroup.all_printed_objects_lst.append(uid)
+                sys.stdout.write('$%d = ' % nid)
             else:
                 print >> sys.stderr, "unexpected message %d" % (msg.cmd,)
 
@@ -168,15 +189,20 @@
         msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
         self.total_stop_points = msg.arg2
         child.expect_ready()
+        self.initial_uid = child.currently_created_objects
 
         self.active = child
         self.paused = {1: child.clone()}     # {time: subprocess}
         self.all_breakpoints = AllBreakpoints()
-        self.all_printed_objects = []
+        self.all_printed_objects = {}
+        self.all_printed_objects_lst = []
 
     def get_current_time(self):
         return self.active.current_time
 
+    def get_currently_created_objects(self):
+        return self.active.currently_created_objects
+
     def _check_current_time(self, time):
         assert self.get_current_time() == time
         self.active.send(Message(CMD_FORWARD, 0))
@@ -186,6 +212,19 @@
         return self.total_stop_points
 
     def get_next_clone_time(self):
+        # if 'active' has more printed_objects than the next process
+        # already in 'paused', then we re-clone 'active'.
+        cur_time = self.get_current_time()
+        future = [time for time in self.paused if time > cur_time]
+        if future:
+            for futime in sorted(future):
+                if (self.paused[futime].printed_objects !=
+                        frozenset(self.all_printed_objects_lst)):
+                    # 'futime' is the time of the first "future" childs
+                    # with an incomplete 'printed_objects'.  This will
+                    # be re-cloned.
+                    return futime
+        #
         if len(self.paused) >= self.MAX_SUBPROCESSES:
             next_time = self.total_stop_points + 1
         else:
@@ -228,6 +267,8 @@
                 elif bkpt:
                     raise bkpt
                 steps -= rel_next_clone
+            if self.active.current_time in self.paused:
+                self.paused[self.active.current_time].close()
             clone = self.active.clone()
             self.paused[clone.current_time] = clone
         bkpt = self.active.forward(steps, breakpoint_mode)
@@ -310,13 +351,69 @@
         for subp in [self.active] + self.paused.values():
             subp.close()
 
-    def print_cmd(self, expression):
+    def ensure_printed_objects(self, uids):
+        """Ensure that all the given unique_ids are loaded in the active
+        child, if necessary by forking another child from earlier.
+        """
+        import pdb;pdb.set_trace()
+        initial_time = self.get_current_time()
+        must_go_forward_again = False
+        while True:
+            uid_limit = self.get_currently_created_objects()
+            missing_uids = [uid for uid in uids
+                                if uid < uid_limit
+                                   and uid not in self.active.printed_objects]
+            if not missing_uids:
+                break
+
+            # we need to start with an older fork
+            start_time = self.get_current_time()
+            stop_time = max(time for time in self.paused if time < start_time)
+            self._resume(stop_time)
+            must_go_forward_again = True
+
+        # No missing_uids left: all uids are either already in
+        # self.active.printed_objects, or in the future.
+        future_uids = [uid for uid in uids if uid >= uid_limit]
+        if not must_go_forward_again:
+            assert not future_uids
+        else:
+            future_uids.sort()
+            pack_uids = [struct.pack('q', uid) for uid in future_uids]
+            self.active.send(Message(CMD_FUTUREIDS, extra=''.join(pack_uids)))
+            self.active.expect_ready()
+            self.active.printed_objects = (
+                self.active.printed_objects.union(future_uids))
+            self.go_forward(initial_time - self.get_current_time(),
+                            breakpoint_mode='i')
+        assert self.active.printed_objects.issuperset(uids)
+
+    def print_cmd(self, expression, nids=[]):
         """Print an expression.
         """
+        if nids:
+            uids = []
+            for nid in nids:
+                try:
+                    uid = self.all_printed_objects_lst[nid]
+                except IndexError:
+                    print >> sys.stderr, ("no print command printed any "
+                                          "value for '$%d'" % nid)
+                    return
+                if uid >= self.get_currently_created_objects():
+                    print >> sys.stderr, ("'$%d' refers to an object that is "
+                                          "only created later in time" % nid)
+                    return
+                uids.append(uid)
+            self.ensure_printed_objects(uids)
+        #
         self.active.tainted = True
-        next_nid = len(self.all_printed_objects)
-        self.active.send(Message(CMD_PRINT, next_nid, extra=expression))
-        self.active.print_text_answer()
+        for nid in nids:
+            uid = self.all_printed_objects_lst[nid]
+            self.active.send(Message(CMD_ATTACHID, nid, uid))
+            self.active.expect_ready()
+        self.active.send(Message(CMD_PRINT, extra=expression))
+        self.active.print_text_answer(pgroup=self)
 
     def show_backtrace(self):
         """Show the backtrace.

From pypy.commits at gmail.com  Tue Jun 21 12:56:38 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 09:56:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <576971c6.871a1c0a.9a011.689f@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85315:fb39f32b33f1
Date: 2016-06-21 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/fb39f32b33f1/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
 import __pypy__
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -315,13 +315,28 @@
 
  - ``complex``
 
+ - ``str`` (empty or single-character strings only)
+
+ - ``unicode`` (empty or single-character strings only)
+
+ - ``tuple`` (empty tuples only)
+
+ - ``frozenset`` (empty frozenset only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
-Notably missing from the list above are ``str`` and ``unicode``.  If your
-code relies on comparing strings with ``is``, then it might break in PyPy.
+Note that strings of length 2 or greater can be equal without being
+identical.  Similarly, ``x is (2,)`` is not necessarily true even if
+``x`` contains a tuple and ``x == (2,)``.  The uniqueness rules apply
+only to the particular cases described above.  The ``str``, ``unicode``,
+``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a
+test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was
+equal to ``"?"`` or ``()``.  The new behavior added in PyPy 5.4 is
+closer to CPython's, which caches precisely the empty tuple/frozenset,
+and (generally but not always) the strings and unicodes of length <= 1.
 
 Note that for floats there "``is``" only one object per "bit pattern"
 of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
    release-5.1.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
    whatsnew-5.0.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.3.1.rst
@@ -0,0 +1,41 @@
+==========
+PyPy 5.3.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.3.0, released last week,
+due to issues_ reported by users.
+
+Thanks to those who reported the issues.
+
+.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,10 @@
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
 
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
 .. pull request #455
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 
@@ -31,3 +35,10 @@
 Simplify handling of interp-level tests and make it more forward-
 compatible.
 
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst
@@ -0,0 +1,15 @@
+===========================
+What's new in PyPy2.7 5.3.1
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3.0
+.. startrev: f4d726d1a010
+
+
+A bug-fix release, merging these changes:
+
+  * Add include guards to pymem.h, fixes issue #2321
+
+  * Make vmprof build on OpenBSD, from pull request #456
+
+  * Fix ``bytearray('').replace('a', 'ab')``, issue #2324
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1212,8 +1212,6 @@
         cpyext_type_init = self.cpyext_type_init
         self.cpyext_type_init = None
         for pto, w_type in cpyext_type_init:
-            if space.is_w(w_type, space.w_str):
-                pto.c_tp_itemsize = 1
             finish_type_1(space, pto)
             finish_type_2(space, pto, w_type)
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -16,7 +16,7 @@
 ## -----------
 ##
 ## PyBytes_AsString() must return a (non-movable) pointer to the underlying
-## buffer, whereas pypy strings are movable.  C code may temporarily store
+## ob_sval, whereas pypy strings are movable.  C code may temporarily store
 ## this address and use it, as long as it owns a reference to the PyObject.
 ## There is no "release" function to specify that the pointer is not needed
 ## any more.
@@ -28,7 +28,7 @@
 ## --------
 ##
 ## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char buffer; it may be NULL.
+## char ob_sval; it may be NULL.
 ##
 ## - A string allocated by pypy will be converted into a PyBytesObject with a
 ##   NULL buffer.  The first time PyBytes_AsString() is called, memory is
@@ -41,6 +41,9 @@
 ##   the pypy string is created, and added to the global map of tracked
 ##   objects.  The buffer is then supposed to be immutable.
 ##
+##-  A buffer obtained from PyBytes_AS_STRING() could be mutable iff
+##   there is no corresponding pypy object for the string
+##
 ## - _PyBytes_Resize() works only on not-yet-pypy'd strings, and returns a
 ##   similar object.
 ##
@@ -53,7 +56,7 @@
 PyBytesObjectStruct = lltype.ForwardReference()
 PyBytesObject = lltype.Ptr(PyBytesObjectStruct)
 PyBytesObjectFields = PyVarObjectFields + \
-    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP))
+    (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char)))
 cpython_struct("PyBytesObject", PyBytesObjectFields, PyBytesObjectStruct)
 
 @bootstrap_function
@@ -69,44 +72,43 @@
 
 def new_empty_str(space, length):
     """
-    Allocate a PyBytesObject and its buffer, but without a corresponding
-    interpreter object.  The buffer may be mutated, until bytes_realize() is
+    Allocate a PyBytesObject and its ob_sval, but without a corresponding
+    interpreter object.  The ob_sval may be mutated, until bytes_realize() is
     called.  Refcount of the result is 1.
     """
     typedescr = get_typedescr(space.w_bytes.layout.typedef)
     py_obj = typedescr.allocate(space, space.w_bytes)
     py_str = rffi.cast(PyBytesObject, py_obj)
-
-    buflen = length + 1
-    py_str.c_ob_size = length
-    py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+    py_str.c_ob_shash = -1
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED
     return py_str
 
 def bytes_attach(space, py_obj, w_obj):
     """
-    Fills a newly allocated PyBytesObject with the given string object. The
-    buffer must not be modified.
+    Copy RPython string object contents to a PyBytesObject. The
+    c_ob_sval must not be modified.
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
-    py_str.c_ob_size = len(space.str_w(w_obj))
-    py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
+    s = space.str_w(w_obj)
+    if py_str.c_ob_size  < len(s):
+        raise oefmt(space.w_ValueError,
+            "bytes_attach called on object with ob_size %d but trying to store %d",
+            py_str.c_ob_size, len(s)) 
+    rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+    py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
 
 def bytes_realize(space, py_obj):
     """
-    Creates the string in the interpreter. The PyBytesObject buffer must not
+    Creates the string in the interpreter. The PyBytesObject ob_sval must not
     be modified after this call.
     """
     py_str = rffi.cast(PyBytesObject, py_obj)
-    if not py_str.c_buffer:
-        py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1,
-                                    flavor='raw', zero=True)
-    s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size)
-    w_obj = space.wrapbytes(s)
+    s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(W_BytesObject, w_type)
+    w_obj.__init__(s)
     py_str.c_ob_shash = space.hash_w(w_obj)
     py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
     track_reference(space, py_obj, w_obj)
@@ -116,9 +118,6 @@
 def bytes_dealloc(space, py_obj):
     """Frees allocated PyBytesObject resources.
     """
-    py_str = rffi.cast(PyBytesObject, py_obj)
-    if py_str.c_buffer:
-        lltype.free(py_str.c_buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -139,36 +138,45 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyBytes_AsString(space, ref):
+    return _PyBytes_AsString(space, ref)
+
+def _PyBytes_AsString(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
         pass    # typecheck returned "ok" without forcing 'ref' at all
     elif not PyBytes_Check(space, ref):   # otherwise, use the alternate way
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.bytes_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    return ref_str.c_buffer
+    if not pyobj_has_w_obj(ref):
+        # XXX Force the ref?
+        bytes_realize(space, ref)
+    return ref_str.c_ob_sval
+
+ at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
+def PyBytes_AS_STRING(space, void_ref):
+    ref = rffi.cast(PyObject, void_ref)
+    # if no w_str is associated with this ref,
+    # return the c-level ptr as RW
+    if not pyobj_has_w_obj(ref):
+        py_str = rffi.cast(PyBytesObject, ref)
+        return py_str.c_ob_sval
+    return _PyBytes_AsString(space, ref)
 
 @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1)
-def PyBytes_AsStringAndSize(space, ref, buffer, length):
+def PyBytes_AsStringAndSize(space, ref, data, length):
     if not PyBytes_Check(space, ref):
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
+    if not pyobj_has_w_obj(ref):
+        # force the ref
+        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not ref_str.c_buffer:
-        # copy string buffer
-        w_str = from_ref(space, ref)
-        s = space.bytes_w(w_str)
-        ref_str.c_buffer = rffi.str2charp(s)
-    buffer[0] = ref_str.c_buffer
+    data[0] = ref_str.c_ob_sval
     if length:
         length[0] = ref_str.c_ob_size
     else:
         i = 0
-        while ref_str.c_buffer[i] != '\0':
+        while ref_str.c_ob_sval[i] != '\0':
             i += 1
         if i != ref_str.c_ob_size:
             raise oefmt(space.w_TypeError,
@@ -197,10 +205,10 @@
     set to NULL, a memory exception is set, and -1 is returned.
     """
     # XXX always create a new string so far
-    py_str = rffi.cast(PyBytesObject, ref[0])
-    if not py_str.c_buffer:
+    if pyobj_has_w_obj(ref[0]):
         raise oefmt(space.w_SystemError,
                     "_PyBytes_Resize called on already created string")
+    py_str = rffi.cast(PyBytesObject, ref[0])
     try:
         py_newstr = new_empty_str(space, newsize)
     except MemoryError:
@@ -212,7 +220,7 @@
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newstr.c_buffer[i] = py_str.c_buffer[i]
+        py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newstr)
     return 0
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.1-alpha0"
-#define PYPY_VERSION_NUM  0x05030100
+#define PYPY_VERSION "5.3.2-alpha0"
+#define PYPY_VERSION_NUM  0x05030200
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
     CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-    INTERPLEVEL_API)
+    INTERPLEVEL_API, PyVarObject)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.objectobject import W_ObjectObject
@@ -47,13 +47,16 @@
             size = pytype.c_tp_basicsize
         else:
             size = rffi.sizeof(self.basestruct)
-        if itemcount and w_type is not space.w_str:
+        if pytype.c_tp_itemsize:
             size += itemcount * pytype.c_tp_itemsize
         assert size >= rffi.sizeof(PyObject.TO)
         buf = lltype.malloc(rffi.VOIDP.TO, size,
                             flavor='raw', zero=True,
                             add_memory_pressure=True)
         pyobj = rffi.cast(PyObject, buf)
+        if pytype.c_tp_itemsize:
+            pyvarobj = rffi.cast(PyVarObject, pyobj)
+            pyvarobj.c_ob_size = itemcount
         pyobj.c_ob_refcnt = 1
         #pyobj.c_ob_pypy_link should get assigned very quickly
         pyobj.c_ob_type = pytype
@@ -152,13 +155,18 @@
 class InvalidPointerException(Exception):
     pass
 
-def create_ref(space, w_obj, itemcount=0):
+def create_ref(space, w_obj):
     """
     Allocates a PyObject, and fills its fields with info from the given
     interpreter object.
     """
     w_type = space.type(w_obj)
+    pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
     typedescr = get_typedescr(w_obj.typedef)
+    if pytype.c_tp_itemsize != 0:
+        itemcount = space.len_w(w_obj) # PyStringObject and subclasses
+    else:
+        itemcount = 0
     py_obj = typedescr.allocate(space, w_type, itemcount=itemcount)
     track_reference(space, py_obj, w_obj)
     #
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py
--- a/pypy/module/cpyext/test/test_bytearrayobject.py
+++ b/pypy/module/cpyext/test/test_bytearrayobject.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+
 class AppTestStringObject(AppTestCpythonExtensionBase):
     def test_basic(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -113,7 +113,15 @@
                 Py_INCREF(obj);
                 return obj;
              """),
+            ('alloc_rw', "METH_NOARGS",
+             '''
+                PyObject *obj = _PyObject_NewVar(&PyBytes_Type, 10);
+                memcpy(PyBytes_AS_STRING(obj), "works", 6);
+                return (PyObject*)obj;
+             '''),
             ])
+        s = module.alloc_rw()
+        assert s == b'works' + b'\x00' * 5
         s = module.tpalloc()
         assert s == b'\x00' * 10
 
@@ -194,22 +202,22 @@
     def test_bytes_resize(self, space, api):
         py_str = new_empty_str(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_str.c_buffer[0] = 'a'
-        py_str.c_buffer[1] = 'b'
-        py_str.c_buffer[2] = 'c'
+        py_str.c_ob_sval[0] = 'a'
+        py_str.c_ob_sval[1] = 'b'
+        py_str.c_ob_sval[2] = 'c'
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyBytes_Resize(ar, 3)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 3
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[3] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[3] == '\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_str)
         api._PyBytes_Resize(ar, 10)
         py_str = rffi.cast(PyBytesObject, ar[0])
         assert py_str.c_ob_size == 10
-        assert py_str.c_buffer[1] == 'b'
-        assert py_str.c_buffer[10] == '\x00'
+        assert py_str.c_ob_sval[1] == 'b'
+        assert py_str.c_ob_sval[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -142,7 +142,7 @@
                     2000, 6, 6, 6, 6, 6, 6, Py_None,
                     PyDateTimeAPI->DateTimeType);
              """),
-        ])
+        ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.new_date() == datetime.date(2000, 6, 6)
         assert module.new_time() == datetime.time(6, 6, 6, 6)
@@ -241,6 +241,9 @@
                  PyObject* obj = PyDelta_FromDSU(6, 6, 6);
                  PyDateTime_Delta* delta = (PyDateTime_Delta*)obj;
 
+#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000
+                 // These macros are only defined in CPython 3.x and PyPy.
+                 // See: http://bugs.python.org/issue13727
                  PyDateTime_DELTA_GET_DAYS(obj);
                  PyDateTime_DELTA_GET_DAYS(delta);
 
@@ -249,10 +252,10 @@
 
                  PyDateTime_DELTA_GET_MICROSECONDS(obj);
                  PyDateTime_DELTA_GET_MICROSECONDS(delta);
-
+#endif
                  return obj;
              """),
-            ])
+            ], prologue='#include "datetime.h"\n')
         import datetime
         assert module.test_date_macros() == datetime.date(2000, 6, 6)
         assert module.test_datetime_macros() == datetime.datetime(
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -875,7 +875,7 @@
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
-        # on cpython, the size changes (6 bytes added)
+        import sys
         module = self.import_module(name='foo')
         size = module.size_of_instances(module.fooType)
         class f1(object):
@@ -885,7 +885,11 @@
         class bar(f1, f2):
             pass
         assert bar.__base__ is f2
-        assert module.size_of_instances(bar) == size
+        # On cpython, the size changes.
+        if '__pypy__' in sys.builtin_module_names:
+            assert module.size_of_instances(bar) == size
+        else:
+            assert module.size_of_instances(bar) >= size
 
     def test_app_cant_subclass_two_types(self):
         module = self.import_module(name='foo')
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -142,6 +142,7 @@
     ref = rffi.cast(PyTupleObject, ref)
     size = ref.c_ob_size
     if index < 0 or index >= size:
+        decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
     ref.c_ob_item[index] = py_obj    # consumes a reference
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -563,6 +563,8 @@
         pto.c_tp_dealloc = llhelper(
             subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
+    if space.is_w(w_type, space.w_str):
+        pto.c_tp_itemsize = 1
     # buffer protocol
     if space.is_w(w_type, space.w_str):
         setup_bytes_buffer_procs(space, pto)
@@ -602,6 +604,8 @@
     if pto.c_tp_base:
         if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize:
             pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize
+        if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
+            pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
     # will be filled later on with the correct value
     # may not be 0
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -72,7 +72,10 @@
     """
     py_uni = rffi.cast(PyUnicodeObject, py_obj)
     s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_length)
-    w_obj = space.wrap(s)
+    w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
+    w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
+    w_obj.__init__(s)
+    py_uni.c_hash = space.hash_w(w_obj)
     track_reference(space, py_obj, w_obj)
     return w_obj
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -163,14 +163,10 @@
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
             guard_true(i32, descr=...)
-            i34 = getarrayitem_raw_i(#, #, descr=)  # XXX what are these?
-            guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i35 = getarrayitem_raw_i(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, 8)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, #, descr=...)                  # XXX
             jump(..., descr=...)
         """)
 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 3, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 3, 2, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -13,6 +13,7 @@
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
 class W_AbstractBytesObject(W_Root):
@@ -25,12 +26,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.bytes_w(self) is space.bytes_w(w_other)
+        s1 = space.str_w(self)
+        s2 = space.str_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.bytes_w(self)))
+        s = space.str_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ord(s[0])     # base values 0-255
+            else:
+                base = 256           # empty string: base value 256
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def descr_add(self, space, w_other):
         """x.__add__(y) <==> x+y"""
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -6,6 +6,7 @@
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 from rpython.rlib.objectmodel import r_dict
 from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash
@@ -562,6 +563,23 @@
 class W_FrozensetObject(W_BaseSetObject):
     hash = 0
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_FrozensetObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty frozensets are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty frozenset: base value 259
+        uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def _newobj(self, space, w_iterable):
         """Make a new frozenset by taking ownership of 'w_iterable'.""" 
         return W_FrozensetObject(space, w_iterable)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -182,7 +182,7 @@
         else:
             res = count(value, sub, start, end)
             assert res >= 0
-            return space.wrap(max(res, 0))
+        return space.wrap(res)
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -183,37 +183,54 @@
     def test_id_on_strs(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        u = "a"
-        assert id(self.unwrap_wrap_unicode(u)) == id(u)
-        s = b"a"
-        assert id(self.unwrap_wrap_bytes(s)) == id(s)
+        for u in [u"", u"a", u"aa"]:
+            assert id(self.unwrap_wrap_unicode(u)) == id(u)
+            s = str(u)
+            assert id(self.unwrap_wrap_str(s)) == id(s)
+        #
+        assert id('') == (256 << 4) | 11     # always
+        assert id(u'') == (257 << 4) | 11
+        assert id('a') == (ord('a') << 4) | 11
+        assert id(u'\u1234') == ((~0x1234) << 4) | 11
+
+    def test_id_of_tuples(self):
+        l = []
+        x = (l,)
+        assert id(x) != id((l,))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(()) == (258 << 4) | 11     # always
+
+    def test_id_of_frozensets(self):
+        x = frozenset([4])
+        assert id(x) != id(frozenset([4]))          # no caching at all
+        if self.appdirect:
+            skip("cannot run this test as apptest")
+        assert id(frozenset()) == (259 << 4) | 11     # always
+        assert id(frozenset([])) == (259 << 4) | 11   # always
 
     def test_identity_vs_id_primitives(self):
-        if self.cpython_apptest:
-            skip("cpython behaves differently")
         import sys
-        l = list(range(-10, 10))
-        for i in range(10):
+        l = range(-10, 10, 2)
+        for i in [0, 1, 3]:
             l.append(float(i))
             l.append(i + 0.1)
+            l.append(long(i))
             l.append(i + sys.maxsize)
             l.append(i - sys.maxsize)
             l.append(i + 1j)
             l.append(i - 1j)
             l.append(1 + i * 1j)
             l.append(1 - i * 1j)
-            s = str(i)
-            l.append(s)
-            u = bytes(s, 'ascii')
-            l.append(u)
+            l.append((i,))
+            l.append(frozenset([i]))
         l.append(-0.0)
         l.append(None)
         l.append(True)
         l.append(False)
-        s = "s"
-        l.append(s)
-        s = b"s"
-        l.append(s)
+        l.append(())
+        l.append(tuple([]))
+        l.append(frozenset())
 
         for i, a in enumerate(l):
             for b in l[i:]:
@@ -224,21 +241,18 @@
     def test_identity_vs_id_str(self):
         if self.appdirect:
             skip("cannot run this test as apptest")
-        import sys
-        l = list(range(-10, 10))
-        for i in range(10):
-            s = bytes(i)
+        l = []
+        def add(s, u):
             l.append(s)
-            l.append(self.unwrap_wrap_bytes(s))
-            u = str(s)
+            l.append(self.unwrap_wrap_str(s))
+            l.append(s[:1] + s[1:])
             l.append(u)
             l.append(self.unwrap_wrap_unicode(u))
-        s = b"s"
-        l.append(s)
-        l.append(self.unwrap_wrap_bytes(s))
-        s = "s"
-        l.append(s)
-        l.append(self.unwrap_wrap_unicode(s))
+            l.append(u[:1] + u[1:])
+        for i in range(3, 18):
+            add(str(i), unicode(i))
+        add(b"s", u"s")
+        add(b"", u"")
 
         for i, a in enumerate(l):
             for b in l[i:]:
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -9,7 +9,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop,
     normalize_simple_slice)
-from pypy.objspace.std.util import negate
+from pypy.objspace.std.util import negate, IDTAG_SPECIAL, IDTAG_SHIFT
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import intmask
@@ -38,6 +38,23 @@
 class W_AbstractTupleObject(W_Root):
     __slots__ = ()
 
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractTupleObject):
+            return False
+        if self is w_other:
+            return True
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return False
+        # empty tuples are unique-ified
+        return 0 == w_other.length() == self.length()
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class or self.length() > 0:
+            return None
+        # empty tuple: base value 258
+        uid = (258 << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
+
     def __repr__(self):
         """representation for debugging purposes"""
         reprlist = [repr(w_item) for w_item in self.tolist()]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -17,6 +17,7 @@
 from pypy.objspace.std import newformat
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'encode_object', 'decode_object',
            'unicode_from_object', 'unicode_to_decimal_w']
@@ -51,12 +52,26 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        return space.unicode_w(self) is space.unicode_w(w_other)
+        s1 = space.unicode_w(self)
+        s2 = space.unicode_w(w_other)
+        if len(s2) > 1:
+            return s1 is s2
+        else:            # strings of len <= 1 are unique-ified
+            return s1 == s2
 
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        return space.wrap(compute_unique_id(space.unicode_w(self)))
+        s = space.unicode_w(self)
+        if len(s) > 1:
+            uid = compute_unique_id(s)
+        else:            # strings of len <= 1 are unique-ified
+            if len(s) == 1:
+                base = ~ord(s[0])      # negative base values
+            else:
+                base = 257       # empty unicode string: base value 257
+            uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
+        return space.wrap(uid)
 
     def unicode_w(self, space):
         return self._value
diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py
--- a/pypy/objspace/std/util.py
+++ b/pypy/objspace/std/util.py
@@ -10,6 +10,12 @@
 IDTAG_FLOAT   = 5
 IDTAG_COMPLEX = 7
 IDTAG_METHOD  = 9
+IDTAG_SPECIAL = 11    # -1 - (-maxunicode-1): unichar
+                      # 0 - 255: char
+                      # 256: empty string
+                      # 257: empty unicode
+                      # 258: empty tuple
+                      # 259: empty frozenset
 
 CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
 BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,9 +1,9 @@
 # Edit these appropriately before running this script
 maj=5
 min=3
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min  # ==OR== release-$maj.$min
+tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState,\
+     VArrayStateInfo, not_virtual, VirtualState,\
      GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
@@ -31,10 +31,10 @@
     def setup_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info = not_virtual(self.cpu, 'r', value)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        self.knownclass_info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        self.knownclass_info2 = not_virtual(self.cpu, 'r', value)
     
     def guards(self, info1, info2, box, runtime_box, expected, inputargs=None):
         if inputargs is None:
@@ -80,7 +80,7 @@
     def test_make_inputargs(self):
         optimizer = FakeOptimizer(self.cpu)
         args = [InputArgInt()]
-        info0 = NotVirtualStateInfo(self.cpu, args[0].type, None)
+        info0 = not_virtual(self.cpu, args[0].type, None)
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == args
         info0.level = LEVEL_CONSTANT
@@ -108,8 +108,8 @@
             assert info1 in state.bad and info2 in state.bad
 
         for BoxType in (InputArgInt, InputArgFloat, InputArgRef):
-            info1 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
-            info2 = NotVirtualStateInfo(self.cpu, BoxType.type, None)
+            info1 = not_virtual(self.cpu, BoxType.type, None)
+            info2 = not_virtual(self.cpu, BoxType.type, None)
             postest(info1, info2)
 
         info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42)
@@ -126,9 +126,9 @@
 
     def test_NotVirtualStateInfo_generalization(self):
         def isgeneral(tp1, info1, tp2, info2):
-            info1 = NotVirtualStateInfo(self.cpu, tp1, info1)
+            info1 = not_virtual(self.cpu, tp1, info1)
             info1.position = 0
-            info2 = NotVirtualStateInfo(self.cpu, tp2, info2)
+            info2 = not_virtual(self.cpu, tp2, info2)
             info2.position = 0
             return VirtualState([info1]).generalization_of(VirtualState([info2]), FakeOptimizer(self.cpu))
 
@@ -166,8 +166,8 @@
         assert not isgeneral('r', value1, 'r', value2)
 
     def test_field_matching_generalization(self):
-        const1 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(1))
-        const2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(2))
+        const1 = not_virtual(self.cpu, 'i', ConstIntBound(1))
+        const2 = not_virtual(self.cpu, 'i', ConstIntBound(2))
         const1.position = const2.position = 1
         self.check_invalid(const1, const2)
         self.check_invalid(const2, const1)
@@ -192,16 +192,16 @@
 
     def test_known_class_generalization(self):
         knownclass1 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info1 = NotVirtualStateInfo(self.cpu, 'r', knownclass1)
+        info1 = not_virtual(self.cpu, 'r', knownclass1)
         info1.position = 0
         knownclass2 = info.InstancePtrInfo(None, ConstPtr(self.myptr))
-        info2 = NotVirtualStateInfo(self.cpu, 'r', knownclass2)
+        info2 = not_virtual(self.cpu, 'r', knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
         knownclass3 = info.InstancePtrInfo(None, ConstPtr(self.myptr2))
-        info3 = NotVirtualStateInfo(self.cpu, 'r', knownclass3)
+        info3 = not_virtual(self.cpu, 'r', knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
         self.check_invalid(info2, info3)
@@ -222,26 +222,26 @@
         #unknown_val = PtrOptValue(self.nodebox)
         #unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
         opt = FakeOptimizer(self.cpu)
-        unknown_info = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info = not_virtual(self.cpu, 'r', None)
 
-        nonnull_info = NotVirtualStateInfo(self.cpu, 'r', info.NonNullPtrInfo())
+        nonnull_info = not_virtual(self.cpu, 'r', info.NonNullPtrInfo())
 
         classbox1 = self.cpu.ts.cls_of_box(ConstPtr(self.nodeaddr))
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox1))
+        knownclass_info = not_virtual(self.cpu, 'r',
+                                      info.InstancePtrInfo(None, classbox1))
         classbox2 = self.cpu.ts.cls_of_box(ConstPtr(self.node2addr))
-        knownclass2_info = NotVirtualStateInfo(self.cpu, 'r',
-                                    info.InstancePtrInfo(None, classbox2))
+        knownclass2_info = not_virtual(self.cpu, 'r',
+                                       info.InstancePtrInfo(None, classbox2))
 
-        constant_info = NotVirtualStateInfo(self.cpu, 'i',
-                                            ConstIntBound(1))
-        constant_ptr_info = NotVirtualStateInfo(self.cpu, 'r',
+        constant_info = not_virtual(self.cpu, 'i',
+                                    ConstIntBound(1))
+        constant_ptr_info = not_virtual(self.cpu, 'r',
                                     info.ConstPtrInfo(ConstPtr(self.nodeaddr)))
         constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr))
-        constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val)
-        constclass2_info = NotVirtualStateInfo(self.cpu, 'r',
+        constclass_info = not_virtual(self.cpu, 'r', constclass_val)
+        constclass2_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.node2addr)))
-        constantnull_info = NotVirtualStateInfo(self.cpu, 'r',
+        constantnull_info = not_virtual(self.cpu, 'r',
                     info.ConstPtrInfo(ConstPtr(self.nullptr)))
 
         # unknown unknown
@@ -260,9 +260,10 @@
         self.check_no_guards(unknown_info, knownclass_info)
 
         # unknown constant
-        self.check_no_guards(unknown_info, constant_info,
+        unknown_info_int = not_virtual(self.cpu, 'i', None)
+        self.check_no_guards(unknown_info_int, constant_info,
                              ConstInt(1), ConstIntBound(1))
-        self.check_no_guards(unknown_info, constant_info)
+        self.check_no_guards(unknown_info_int, constant_info)
 
 
         # nonnull unknown
@@ -293,11 +294,11 @@
         const_nonnull = ConstPtr(self.nodeaddr)
         const_nonnull2 = ConstPtr(self.node2addr)
         const_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
-        self.check_no_guards(nonnull_info, constant_info, const_nonnull,
+        self.check_no_guards(nonnull_info, constant_ptr_info, const_nonnull,
                              info.ConstPtrInfo(const_nonnull))
         self.check_invalid(nonnull_info, constantnull_info, const_null,
                            info.ConstPtrInfo(const_null))
-        self.check_no_guards(nonnull_info, constant_info)
+        self.check_no_guards(nonnull_info, constant_ptr_info)
         self.check_invalid(nonnull_info, constantnull_info)
 
 
@@ -392,8 +393,8 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', IntUnbounded())
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', IntUnbounded())
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -408,18 +409,18 @@
         value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(10000))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(10000))
         self.check_invalid(info1, info2)
-        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(11))
+        info1 = not_virtual(self.cpu, 'i', value1)
+        info2 = not_virtual(self.cpu, 'i', ConstIntBound(11))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value1 = info.InstancePtrInfo(None, classbox)
-        info1 = NotVirtualStateInfo(self.cpu, 'r', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        info1 = not_virtual(self.cpu, 'r', value1)
+        info2 = not_virtual(self.cpu, 'r', None)
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []        
@@ -456,18 +457,18 @@
     def test_equal_inputargs(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info1 = not_virtual(self.cpu, 'r', None)
+        unknown_info2 = not_virtual(self.cpu, 'r', None)
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate3.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -494,9 +495,9 @@
     def test_generate_guards_on_virtual_fields_matches_array(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         descr = ArrayDescr(lltype.GcArray(llmemory.GCREF), self.cpu)
@@ -524,9 +525,9 @@
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
@@ -552,9 +553,9 @@
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         structdescr = self.nodesize
@@ -583,9 +584,9 @@
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
         constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
+        innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         NODE = lltype.Struct('NODE', ('x', llmemory.GCREF))
@@ -627,7 +628,7 @@
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info2.fieldstate = [unknown_info1, unknown_info1]
         vstate2 = VirtualState([info2])
@@ -636,9 +637,9 @@
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
         info3 = VirtualStateInfo(ConstInt(42), [1, 2])
-        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
-        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r',
+        unknown_info2 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info3.fieldstate = [unknown_info1, unknown_info2]
         vstate3 = VirtualState([info3])        
@@ -651,7 +652,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -659,7 +660,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -671,7 +672,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -679,7 +680,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -691,7 +692,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -699,24 +700,24 @@
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
         assert not vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
-        
+
     def test_nonvirtual_is_not_virtual(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = NotVirtualStateInfo(self.cpu, 'r', value)
+        info2 = not_virtual(self.cpu, 'r', value)
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
@@ -727,7 +728,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -735,7 +736,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -747,7 +748,7 @@
         info1 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
@@ -755,7 +756,7 @@
         info2 = VArrayStateInfo(42)
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
-        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
+        knownclass_info = not_virtual(self.cpu, 'r', value)
         info2.fieldstate = [knownclass_info]
         vstate2 = VirtualState([info2])
         assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu))
@@ -793,7 +794,7 @@
     def test_crash_varay_clear(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         innervalue1 = info.InstancePtrInfo(None, classbox)
-        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
+        innerinfo1 = not_virtual(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
         innerinfo1.position_in_notvirtuals = 0
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -350,40 +350,23 @@
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
 
+
+def not_virtual(cpu, type, info):
+    if type == 'i':
+        return NotVirtualStateInfoInt(cpu, type, info)
+    if type == 'r':
+        return NotVirtualStateInfoPtr(cpu, type, info)
+    return NotVirtualStateInfo(cpu, type, info)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
-    lenbound = None
-    intbound = None
     level = LEVEL_UNKNOWN
     constbox = None
-    known_class = None
-    
+
     def __init__(self, cpu, type, info):
         if info and info.is_constant():
             self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
-            if type == 'r':
-                self.known_class = info.get_known_class(cpu)
-            elif type == 'i':
-                self.intbound = info
-        elif type == 'r':
-            if info:
-                self.known_class = info.get_known_class(cpu)
-                if self.known_class:
-                    self.level = LEVEL_KNOWNCLASS
-                elif info.is_nonnull():
-                    self.level = LEVEL_NONNULL
-                self.lenbound = info.getlenbound(None)
-        elif type == 'i':
-            if isinstance(info, IntBound):
-                if info.lower < MININT / 2:
-                    info.lower = MININT
-                if info.upper > MAXINT / 2:
-                    info.upper = MAXINT
-                self.intbound = info
-        elif type == 'f':
-            if info and info.is_constant():
-                self.level = LEVEL_CONSTANT
-                self.constbox = info.getconst()
 
     def is_const(self):
         return self.constbox is not None
@@ -394,84 +377,15 @@
     def _generate_guards(self, other, box, runtime_box, state):
         # XXX This will always retrace instead of forcing anything which
         # might be what we want sometimes?
-        if not isinstance(other, NotVirtualStateInfo):
-            raise VirtualStatesCantMatch(
-                    'The VirtualStates does not match as a ' +
-                    'virtual appears where a pointer is needed ' +
-                    'and it is too late to force it.')
-
-
         extra_guards = state.extra_guards
-        cpu = state.cpu
-        if self.lenbound:
-            if other.lenbound is None:
-                other_bound = IntLowerBound(0)
-            else:
-                other_bound = other.lenbound
-            if not self.lenbound.contains_bound(other_bound):
-                raise VirtualStatesCantMatch("length bound does not match")
-
         if self.level == LEVEL_UNKNOWN:
-            # confusingly enough, this is done also for pointers
-            # which have the full range as the "bound", so it always works
-            return self._generate_guards_intbounds(other, box, runtime_box,
-                                                   extra_guards,
-                                                   state.optimizer)
-
-        # the following conditions often peek into the runtime value that the
-        # box had when tracing. This value is only used as an educated guess.
-        # It is used here to choose between either emitting a guard and jumping
-        # to an existing compiled loop or retracing the loop. Both alternatives
-        # will always generate correct behaviour, but performance will differ.
-        elif self.level == LEVEL_NONNULL:
-            if other.level == LEVEL_UNKNOWN:
-                if runtime_box is not None and runtime_box.nonnull():
-                    op = ResOperation(rop.GUARD_NONNULL, [box])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other not known to be nonnull")
-            elif other.level == LEVEL_NONNULL:
-                return
-            elif other.level == LEVEL_KNOWNCLASS:
-                return # implies nonnull
-            else:
-                assert other.level == LEVEL_CONSTANT
-                assert other.constbox
-                if not other.constbox.nonnull():
-                    raise VirtualStatesCantMatch("constant is null")
-                return
-
-        elif self.level == LEVEL_KNOWNCLASS:
-            if other.level == LEVEL_UNKNOWN:
-                if (runtime_box and runtime_box.nonnull() and
-              self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_NONNULL:
-                if (runtime_box and self.known_class.same_constant(
-                        cpu.ts.cls_of_box(runtime_box))):
-                    op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
-                    extra_guards.append(op)
-                    return
-                else:
-                    raise VirtualStatesCantMatch("other's class is unknown")
-            elif other.level == LEVEL_KNOWNCLASS:
-                if self.known_class.same_constant(other.known_class):
-                    return
-                raise VirtualStatesCantMatch("classes don't match")
-            else:
-                assert other.level == LEVEL_CONSTANT
-                if (other.constbox.nonnull() and
-                        self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))):
-                    return
-                else:
-                    raise VirtualStatesCantMatch("classes don't match")
-
+            return self._generate_guards_unkown(other, box, runtime_box,
+                                                extra_guards,
+                                                state)
         else:
+            if not isinstance(other, NotVirtualStateInfo):
+                raise VirtualStatesCantMatch(
+                        'comparing a constant against something that is a virtual')
             assert self.level == LEVEL_CONSTANT
             if other.level == LEVEL_CONSTANT:
                 if self.constbox.same_constant(other.constbox):
@@ -485,19 +399,9 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, box, runtime_box, extra_guards,
-                                   optimizer):
-        if self.intbound is None:
-            return
-        if self.intbound.contains_bound(other.intbound):
-            return
-        if (runtime_box is not None and
-            self.intbound.contains(runtime_box.getint())):
-            # this may generate a few more guards than needed, but they are
-            # optimized away when emitting them
-            self.intbound.make_guards(box, extra_guards, optimizer)
-            return
-        raise VirtualStatesCantMatch("intbounds don't match")
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                state):
+        return
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         if self.level == LEVEL_CONSTANT:
@@ -553,8 +457,145 @@
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
 
-        debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
-                    ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
+        result = indent + mark + 'NotVirtualStateInfo(%d' % self.position + ', ' + l
+        extra = self._extra_repr()
+        if extra:
+            result += ', ' + extra
+        result += lb + ')'
+        debug_print(result)
+
+class NotVirtualStateInfoInt(NotVirtualStateInfo):
+    intbound = None
+
+    def __init__(self, cpu, type, info):
+        NotVirtualStateInfo.__init__(self, cpu, type, info)
+        assert type == 'i'
+        if isinstance(info, IntBound):
+            if info.lower < MININT / 2:
+                info.lower = MININT
+            if info.upper > MAXINT / 2:
+                info.upper = MAXINT
+            self.intbound = info
+
+    def _generate_guards_unkown(self, other, box, runtime_box, extra_guards,
+                                state):
+        other_intbound = None
+        if isinstance(other, NotVirtualStateInfoInt):
+            other_intbound = other.intbound
+        if self.intbound is None:
+            return
+        if self.intbound.contains_bound(other_intbound):
+            return
+        if (runtime_box is not None and
+            self.intbound.contains(runtime_box.getint())):
+            # this may generate a few more guards than needed, but they are
+            # optimized away when emitting them
+            self.intbound.make_guards(box, extra_guards, state.optimizer)
+            return
+        raise VirtualStatesCantMatch("intbounds don't match")
+
+    def _extra_repr(self):
+        return self.intbound.__repr__()
+
+
+class NotVirtualStateInfoPtr(NotVirtualStateInfo):
+    lenbound = None
+    known_class = None
+
+    def __init__(self, cpu, type, info):
+        if info:
+            self.known_class = info.get_known_class(cpu)
+            if self.known_class:
+                self.level = LEVEL_KNOWNCLASS
+            elif info.is_nonnull():
+                self.level = LEVEL_NONNULL
+            self.lenbound = info.getlenbound(None)
+        # might set it to LEVEL_CONSTANT
+        NotVirtualStateInfo.__init__(self, cpu, type, info)
+
+    def _generate_guards(self, other, box, runtime_box, state):
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch(
+                    'The VirtualStates does not match as a ' +
+                    'virtual appears where a pointer is needed ' +
+                    'and it is too late to force it.')
+        extra_guards = state.extra_guards
+        if self.lenbound:
+            if other.lenbound is None:
+                other_bound = IntLowerBound(0)
+            else:
+                other_bound = other.lenbound
+            if not self.lenbound.contains_bound(other_bound):
+                raise VirtualStatesCantMatch("length bound does not match")
+        if self.level == LEVEL_NONNULL:
+            return self._generate_guards_nonnull(other, box, runtime_box,
+                                                 extra_guards,
+                                                 state)
+        elif self.level == LEVEL_KNOWNCLASS:
+            return self._generate_guards_knownclass(other, box, runtime_box,
+                                                    extra_guards,
+                                                    state)
+        return NotVirtualStateInfo._generate_guards(self, other, box,
+                                                    runtime_box, state)
+
+
+    # the following methods often peek into the runtime value that the
+    # box had when tracing. This value is only used as an educated guess.
+    # It is used here to choose between either emitting a guard and jumping
+    # to an existing compiled loop or retracing the loop. Both alternatives
+    # will always generate correct behaviour, but performance will differ.
+
+    def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
+                                 state):
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!')
+        if other.level == LEVEL_UNKNOWN:
+            if runtime_box is not None and runtime_box.nonnull():
+                op = ResOperation(rop.GUARD_NONNULL, [box])
+                extra_guards.append(op)
+                return
+            else:
+                raise VirtualStatesCantMatch("other not known to be nonnull")
+        elif other.level == LEVEL_NONNULL:
+            pass
+        elif other.level == LEVEL_KNOWNCLASS:
+            pass # implies nonnull
+        else:
+            assert other.level == LEVEL_CONSTANT
+            assert other.constbox
+            if not other.constbox.nonnull():
+                raise VirtualStatesCantMatch("constant is null")
+
+    def _generate_guards_knownclass(self, other, box, runtime_box, extra_guards,
+                                    state):
+        cpu = state.cpu
+        if not isinstance(other, NotVirtualStateInfoPtr):
+            raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!')
+        if other.level == LEVEL_UNKNOWN:
+            if (runtime_box and runtime_box.nonnull() and
+                  self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
+                op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
+                extra_guards.append(op)
+            else:
+                raise VirtualStatesCantMatch("other's class is unknown")
+        elif other.level == LEVEL_NONNULL:
+            if (runtime_box and self.known_class.same_constant(
+                    cpu.ts.cls_of_box(runtime_box))):
+                op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
+                extra_guards.append(op)
+            else:
+                raise VirtualStatesCantMatch("other's class is unknown")
+        elif other.level == LEVEL_KNOWNCLASS:
+            if self.known_class.same_constant(other.known_class):
+                return
+            raise VirtualStatesCantMatch("classes don't match")
+        else:
+            assert other.level == LEVEL_CONSTANT
+            if (other.constbox.nonnull() and
+                    self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))):
+                pass
+            else:
+                raise VirtualStatesCantMatch("classes don't match")
 
 
 class VirtualState(object):
@@ -678,8 +719,8 @@
         return VirtualState(state)
 
     def visit_not_virtual(self, box):
-        return NotVirtualStateInfo(self.optimizer.cpu, box.type,
-                                   self.optimizer.getinfo(box))
+        return not_virtual(self.optimizer.cpu, box.type,
+                           self.optimizer.getinfo(box))
 
     def visit_virtual(self, descr, fielddescrs):
         known_class = ConstInt(descr.get_vtable())

From pypy.commits at gmail.com  Tue Jun 21 13:16:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Jun 2016 10:16:27 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fixes
Message-ID: <5769766b.a91cc20a.d7d7b.4655@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85316:afd4dd224a4e
Date: 2016-06-21 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/afd4dd224a4e/

Log:	fixes

diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -71,7 +71,7 @@
         return ''.join(pieces)
 
     def send(self, msg):
-        print 'SENT:', self.pid, msg
+        #print 'SENT:', self.pid, msg
         binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
                              msg.arg1, msg.arg2, msg.arg3)
         self.control_socket.sendall(binary + msg.extra)
@@ -81,7 +81,7 @@
         cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
         extra = self._recv_all(size)
         msg = Message(cmd, arg1, arg2, arg3, extra)
-        print 'RECV:', self.pid, msg
+        #print 'RECV:', self.pid, msg
         return msg
 
     def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
@@ -355,29 +355,27 @@
         """Ensure that all the given unique_ids are loaded in the active
         child, if necessary by forking another child from earlier.
         """
-        import pdb;pdb.set_trace()
         initial_time = self.get_current_time()
-        must_go_forward_again = False
+        child = self.active
         while True:
-            uid_limit = self.get_currently_created_objects()
+            uid_limit = child.currently_created_objects
             missing_uids = [uid for uid in uids
                                 if uid < uid_limit
-                                   and uid not in self.active.printed_objects]
+                                   and uid not in child.printed_objects]
             if not missing_uids:
                 break
-
-            # we need to start with an older fork
-            start_time = self.get_current_time()
+            # pick the earlier fork
+            start_time = child.current_time
             stop_time = max(time for time in self.paused if time < start_time)
-            self._resume(stop_time)
-            must_go_forward_again = True
+            child = self.paused[stop_time]
 
         # No missing_uids left: all uids are either already in
         # self.active.printed_objects, or in the future.
         future_uids = [uid for uid in uids if uid >= uid_limit]
-        if not must_go_forward_again:
+        if child is self.active:
             assert not future_uids
         else:
+            self._resume(stop_time)
             future_uids.sort()
             pack_uids = [struct.pack('q', uid) for uid in future_uids]
             self.active.send(Message(CMD_FUTUREIDS, extra=''.join(pack_uids)))
@@ -391,25 +389,24 @@
     def print_cmd(self, expression, nids=[]):
         """Print an expression.
         """
+        uids = []
         if nids:
-            uids = []
-            for nid in nids:
+            for nid in set(nids):
                 try:
                     uid = self.all_printed_objects_lst[nid]
                 except IndexError:
-                    print >> sys.stderr, ("no print command printed any "
-                                          "value for '$%d'" % nid)
-                    return
+                    continue
                 if uid >= self.get_currently_created_objects():
-                    print >> sys.stderr, ("'$%d' refers to an object that is "
-                                          "only created later in time" % nid)
-                    return
+                    print >> sys.stderr, (
+                        "note: '$%d' refers to an object that is "
+                        "only created later in time" % nid)
+                    continue
                 uids.append(uid)
             self.ensure_printed_objects(uids)
         #
         self.active.tainted = True
-        for nid in nids:
-            uid = self.all_printed_objects_lst[nid]
+        for uid in uids:
+            nid = self.all_printed_objects[uid]
             self.active.send(Message(CMD_ATTACHID, nid, uid))
             self.active.expect_ready()
         self.active.send(Message(CMD_PRINT, extra=expression))
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -559,6 +559,7 @@
         }
         memcpy(future_ids, extra, cmd->extra_size);
         future_ids[cmd->extra_size / sizeof(uint64_t)] = 0;
+        rpy_revdb.unique_id_break = *future_ids;
     }
     future_next_id = future_ids;
 }
@@ -736,17 +737,22 @@
 RPY_EXTERN
 uint64_t rpy_reverse_db_unique_id_break(void *new_object)
 {
+    uint64_t uid = rpy_revdb.unique_id_seen;
     if (!new_object) {
         fprintf(stderr, "out of memory: allocation failed, cannot continue\n");
         exit(1);
     }
     if (rpy_revdb_commands.rp_alloc) {
+        rpy_revdb_t dinfo;
         save_state();
-        rpy_revdb_commands.rp_alloc(rpy_revdb.unique_id_seen, new_object);
+        disable_io(&dinfo);
+        if (setjmp(jmp_buf_cancel_execution) == 0)
+            rpy_revdb_commands.rp_alloc(uid, new_object);
+        enable_io(&dinfo);
         restore_state();
     }
     rpy_revdb.unique_id_break = *future_next_id++;
-    return rpy_revdb.unique_id_seen;
+    return uid;
 }
 
 

From pypy.commits at gmail.com  Tue Jun 21 14:13:56 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 11:13:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: manually merge default's stringobject.h
 changes to bytesobject.h
Message-ID: <576983e4.54df1c0a.948f9.7e7c@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85317:ed21d8456ad2
Date: 2016-06-21 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/ed21d8456ad2/

Log:	manually merge default's stringobject.h changes to bytesobject.h

diff --git a/pypy/module/cpyext/include/bytesobject.h b/pypy/module/cpyext/include/bytesobject.h
--- a/pypy/module/cpyext/include/bytesobject.h
+++ b/pypy/module/cpyext/include/bytesobject.h
@@ -9,7 +9,6 @@
 #include 
 
 #define PyBytes_GET_SIZE(op) PyBytes_Size(op)
-#define PyBytes_AS_STRING(op) PyBytes_AsString(op)
 
 /*
 Type PyStringObject represents a character string.  An extra zero byte is
@@ -35,24 +34,20 @@
    Together, these sped cpython up by up to 20%, and since they are part of the
    "public" interface PyPy must reimpliment them. */
 
-
-
 typedef struct {
     PyObject_VAR_HEAD
     long ob_shash;
     int ob_sstate;
-    char * buffer; /* change the name from cpython so all non-api c access is thwarted */
+    char ob_sval[1]; 
 
     /* Invariants 
-     * (not relevant in PyPy, all stringobjects are backed by a pypy object)
-     *     buffer contains space for 'ob_size+1' elements.
-     *     buffer[ob_size] == 0.
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
      *     ob_shash is the hash of the string or -1 if not computed yet.
      *     ob_sstate != 0 iff the string object is in stringobject.c's
      *       'interned' dictionary; in this case the two references
      *       from 'interned' to this object are *not counted* in ob_refcnt.
      */
-
 } PyBytesObject;
 
 #define SSTATE_NOT_INTERNED 0

From pypy.commits at gmail.com  Tue Jun 21 14:27:36 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Jun 2016 11:27:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix missing imports
Message-ID: <57698718.c445c20a.9300e.48be@mx.google.com>

Author: Ronan Lamy 
Branch: py3k
Changeset: r85318:d5533cd7171d
Date: 2016-06-21 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/d5533cd7171d/

Log:	fix missing imports

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -6,7 +6,9 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref)
+    make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref,
+    pyobj_has_w_obj)
+from pypy.objspace.std.bytesobject import W_BytesObject
 
 ##
 ## Implementation of PyBytesObject
@@ -93,7 +95,7 @@
     if py_str.c_ob_size  < len(s):
         raise oefmt(space.w_ValueError,
             "bytes_attach called on object with ob_size %d but trying to store %d",
-            py_str.c_ob_size, len(s)) 
+            py_str.c_ob_size, len(s))
     rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
     py_str.c_ob_sval[len(s)] = '\0'
     py_str.c_ob_shash = space.hash_w(w_obj)

From pypy.commits at gmail.com  Tue Jun 21 16:58:25 2016
From: pypy.commits at gmail.com (raffael_t)
Date: Tue, 21 Jun 2016 13:58:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: change handling dicts in astbuilder
Message-ID: <5769aa71.e457c20a.47e40.ffffdec0@mx.google.com>

Author: Raffael Tfirst 
Branch: py3.5
Changeset: r85319:bfeac9d99b15
Date: 2016-06-21 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/bfeac9d99b15/

Log:	change handling dicts in astbuilder

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1136,6 +1136,17 @@
                 raise
             return self.space.call_function(self.space.w_float, w_num_str)
 
+    def handle_dictelement(self, node, i):
+        if node.get_child(i).type == tokens.DOUBLESTAR:
+            key = None
+            value = self.handle_expr(node.get_child(i+1))
+            i += 2
+        else:
+            key = self.handle_expr(node.get_child(i))
+            value = self.handle_expr(node.get_child(i+2))
+            i += 3
+        return [i,key,value]
+    
     def handle_atom(self, atom_node):
         first_child = atom_node.get_child(0)
         first_child_type = first_child.type
@@ -1198,26 +1209,31 @@
             return self.handle_listcomp(second_child)
         elif first_child_type == tokens.LBRACE:
             maker = atom_node.get_child(1)
+            n_maker_children = maker.num_children()
             if maker.type == tokens.RBRACE:
+                # an empty dict
                 return ast.Dict(None, None, atom_node.get_lineno(), atom_node.get_column())
-            n_maker_children = maker.num_children()
-            #import pdb;pdb.set_trace()
-            if n_maker_children == 1 or maker.get_child(1).type == tokens.COMMA:
-                elts = []
-                for i in range(0, n_maker_children, 2):
-                    elts.append(self.handle_expr(maker.get_child(i)))
-                return ast.Set(elts, atom_node.get_lineno(), atom_node.get_column())
-            if maker.get_child(1).type == syms.comp_for:
-                return self.handle_setcomp(maker)
-            if (n_maker_children > 3 and
-                maker.get_child(3).type == syms.comp_for):
-                return self.handle_dictcomp(maker)
-            keys = []
-            values = []
-            for i in range(0, n_maker_children, 4):
-                keys.append(self.handle_expr(maker.get_child(i)))
-                values.append(self.handle_expr(maker.get_child(i + 2)))
-            return ast.Dict(keys, values, atom_node.get_lineno(), atom_node.get_column())
+            else:
+                is_dict = maker.get_child(0).type == tokens.DOUBLESTAR
+                if (n_maker_children == 1 or
+                    (n_maker_children > 1 and
+                     maker.get_child(1).type == tokens.COMMA)):
+                    # a set display
+                    return handle_setdisplay(maker)
+                elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for:
+                    # a set comprehension
+                    return self.handle_setcomp(maker)
+                elif (n_maker_children > 3 - is_dict and
+                      maker.get_child(3-is_dict).type == syms.comp_for):
+                    # a dictionary comprehension
+                    if is_dict:
+                        raise self.error("dict unpacking cannot be used in "
+                                         "dict comprehension", atom_node)
+                    
+                    return self.handle_dictcomp(maker)
+                else:
+                    # a dictionary display
+                    return handle_dictdisplay(maker)
         else:
             raise AssertionError("unknown atom")
 
@@ -1316,11 +1332,19 @@
                            set_maker.get_column())
 
     def handle_dictcomp(self, dict_maker):
-        key = self.handle_expr(dict_maker.get_child(0))
-        value = self.handle_expr(dict_maker.get_child(2))
-        comps = self.comprehension_helper(dict_maker.get_child(3))
+        i = 0
+        dictelement = self.handle_dictelement(dict_maker, i)
+        i = dictelement[0]
+        key = dictelement[1]
+        value = dictelement[2]
+        comps = self.comprehension_helper(dict_maker.get_child(i))
         return ast.DictComp(key, value, comps, dict_maker.get_lineno(),
                             dict_maker.get_column())
+    
+    def handle_dictdisplay(self, node):
+        size = (node.num_children()+1) / 3
+        
+        return ast.Dict(keys, values, node.get_lineno(), node.get_column())
 
     def handle_exprlist(self, exprlist, context):
         exprs = []

From pypy.commits at gmail.com  Wed Jun 22 04:03:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Jun 2016 01:03:57 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Interface tweaks
Message-ID: <576a466d.6a25c20a.614b4.5eb3@mx.google.com>

Author: Armin Rigo 
Branch: reverse-debugger
Changeset: r85320:a80b43cb9d22
Date: 2016-06-22 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/a80b43cb9d22/

Log:	Interface tweaks

diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -25,8 +25,13 @@
 
     def interact(self):
         last_command = 'help'
+        previous_time = None
         while True:
             last_time = self.pgroup.get_current_time()
+            if last_time != previous_time:
+                print
+                self.pgroup.show_backtrace(complete=0)
+                previous_time = last_time
             prompt = '(%d)$ ' % last_time
             sys.stdout.write(prompt)
             sys.stdout.flush()
@@ -55,11 +60,19 @@
     def command_help(self, argument):
         """Display commands summary"""
         print 'Available commands:'
-        for name in dir(self):
-            if name.startswith('command_'):
-                command = name[len('command_'):]
-                docstring = getattr(self, name).__doc__ or 'undocumented'
-                print '\t%-12s %s' % (command, docstring)
+        lst = dir(self)
+        commands = [(name[len('command_'):], getattr(self, name))
+                    for name in lst
+                        if name.startswith('command_')]
+        seen = {}
+        for name, func in commands:
+            seen.setdefault(func, []).append(name)
+        for _, func in commands:
+            if func in seen:
+                names = seen.pop(func)
+                names.sort(key=len, reverse=True)
+                docstring = func.__doc__ or 'undocumented'
+                print '\t%-16s %s' % (', '.join(names), docstring)
 
     def command_quit(self, argument):
         """Exit the debugger"""
@@ -92,6 +105,7 @@
         print ', '.join(lst)
 
     def move_forward(self, steps):
+        self.remove_tainting()
         try:
             self.pgroup.go_forward(steps)
             return True
@@ -123,7 +137,6 @@
     def command_step(self, argument):
         """Run forward ARG steps (default 1)"""
         arg = int(argument or '1')
-        self.remove_tainting()
         self.move_forward(arg)
     command_s = command_step
 
@@ -145,7 +158,6 @@
 
     def command_next(self, argument):
         """Run forward for one step, skipping calls"""
-        self.remove_tainting()
         depth1 = self.pgroup.get_stack_depth()
         if self.move_forward(1):
             depth2 = self.pgroup.get_stack_depth()
@@ -173,11 +185,12 @@
     command_bn = command_bnext
 
     def command_finish(self, argument):
-        self.remove_tainting()
+        """Run forward until the current function finishes"""
         with self._stack_depth_break(self.pgroup.get_stack_depth()):
             self.command_continue('')
 
     def command_bfinish(self, argument):
+        """Run backward until the current function is called"""
         with self._stack_depth_break(self.pgroup.get_stack_depth()):
             self.command_bcontinue('')
 
@@ -201,7 +214,7 @@
 
     def command_backtrace(self, argument):
         """Show the backtrace"""
-        self.pgroup.show_backtrace()
+        self.pgroup.show_backtrace(complete=1)
     command_bt = command_backtrace
 
     def command_locals(self, argument):
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
--- a/rpython/translator/revdb/process.py
+++ b/rpython/translator/revdb/process.py
@@ -412,10 +412,10 @@
         self.active.send(Message(CMD_PRINT, extra=expression))
         self.active.print_text_answer(pgroup=self)
 
-    def show_backtrace(self):
+    def show_backtrace(self, complete=1):
         """Show the backtrace.
         """
-        self.active.send(Message(CMD_BACKTRACE))
+        self.active.send(Message(CMD_BACKTRACE, complete))
         self.active.print_text_answer()
 
     def show_locals(self):
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -28,7 +28,7 @@
         lambda_blip = lambda: blip
 
         def main(argv):
-            revdb.register_debug_command(1, lambda_blip)
+            revdb.register_debug_command(100, lambda_blip)
             for i, op in enumerate(argv[1:]):
                 dbstate.stuff = Stuff()
                 dbstate.stuff.x = i + 1000
@@ -63,8 +63,8 @@
 
     def test_breakpoint_b(self):
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(1, 6, extra='set-breakpoint'))
-        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.send(Message(100, 6, extra='set-breakpoint'))
+        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
         group.active.expect(ANSWER_READY, 1, Ellipsis)
         e = py.test.raises(Breakpoint, group.go_forward, 10, 'b')
         assert e.value.time == 7
@@ -73,8 +73,8 @@
 
     def test_breakpoint_r(self):
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(1, 6, extra='set-breakpoint'))
-        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.send(Message(100, 6, extra='set-breakpoint'))
+        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
         group.active.expect(ANSWER_READY, 1, Ellipsis)
         e = py.test.raises(Breakpoint, group.go_forward, 10, 'r')
         assert e.value.time == 8
@@ -83,7 +83,7 @@
 
     def test_breakpoint_i(self):
         group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(1, 6, extra='set-breakpoint'))
-        group.active.expect(42, 1, -43, -44, 'set-breakpoint')
+        group.active.send(Message(100, 6, extra='set-breakpoint'))
+        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
         group.active.expect(ANSWER_READY, 1, Ellipsis)
         group.go_forward(10, 'i')    # does not raise Breakpoint

From pypy.commits at gmail.com  Wed Jun 22 04:23:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Jun 2016 01:23:39 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Add a link to Building from source
Message-ID: <576a4b0b.a91cc20a.d7d7b.47d7@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r761:0c226313307f
Date: 2016-06-22 10:25 +0200
http://bitbucket.org/pypy/pypy.org/changeset/0c226313307f/

Log:	Add a link to Building from source

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -111,6 +111,7 @@
 version): Ubuntu (PPA), Debian, Homebrew, MacPorts,
 Fedora, Gentoo and Arch are known to package PyPy, with various
 degrees of being up-to-date.
+
  • or translate your own PyPy.
    •
    diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -64,6 +64,8 @@ `Fedora`_, `Gentoo`_ and `Arch`_ are known to package PyPy, with various degrees of being up-to-date. +* or translate_ your own PyPy. + .. _`Ubuntu`: http://packages.ubuntu.com/search?keywords=pypy&searchon=names .. _`PPA`: https://launchpad.net/~pypy/+archive/ppa .. _`Debian`: http://packages.debian.org/sid/pypy From pypy.commits at gmail.com Wed Jun 22 04:37:55 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 22 Jun 2016 01:37:55 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: tweaks Message-ID: <576a4e63.c99dc20a.7b6bc.ffffaa59@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85321:76a8da332a75 Date: 2016-06-22 10:38 +0200 http://bitbucket.org/pypy/pypy/changeset/76a8da332a75/ Log: tweaks diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -22,6 +22,7 @@ if executable is None: executable = fields[1] self.pgroup = ReplayProcessGroup(executable, revdb_log_filename) + self.print_extra_pending_info = None def interact(self): last_command = 'help' @@ -30,6 +31,10 @@ last_time = self.pgroup.get_current_time() if last_time != previous_time: print + if self.print_extra_pending_info: + print self.print_extra_pending_info + self.print_extra_pending_info = None + if last_time != previous_time: self.pgroup.show_backtrace(complete=0) previous_time = last_time prompt = '(%d)$ ' % last_time @@ -115,7 +120,7 @@ def move_backward(self, steps): try: - self.pgroup.go_backward(steps) + self.pgroup.go_backward(steps, ignore_breakpoints=(steps==1)) return True except Breakpoint as b: self.hit_breakpoint(b, backward=True) @@ -123,7 +128,7 @@ def hit_breakpoint(self, b, backward=False): if b.num != -1: - print 'Hit breakpoint %d' % (b.num,) + self.print_extra_pending_info = 'Hit breakpoint %d' % (b.num,) elif backward: b.time -= 1 if self.pgroup.get_current_time() != b.time: diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -277,7 +277,7 @@ if bkpt: raise bkpt - def go_backward(self, steps): + def go_backward(self, steps, ignore_breakpoints=False): """Go backward, for the given number of 'steps' of time. Closes the active process. Implemented as jump_in_time() @@ -285,7 +285,7 @@ """ assert steps >= 0 initial_time = self.get_current_time() - if self.all_breakpoints.is_empty(): + if self.all_breakpoints.is_empty() or ignore_breakpoints: self.jump_in_time(initial_time - steps) else: self._backward_search_forward( @@ -300,7 +300,7 @@ search_start_time = self.get_current_time() time_range_to_search = search_stop_time - search_start_time if time_range_to_search <= 0: - print "[not found]" + print "[search end]" return print "[searching %d..%d]" % (search_start_time, search_stop_time) From pypy.commits at gmail.com Wed Jun 22 06:12:24 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 22 Jun 2016 03:12:24 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <576a6488.cf981c0a.1de07.1cbe@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85322:1e5b6abe3bdb Date: 2016-06-22 12:13 +0200 http://bitbucket.org/pypy/pypy/changeset/1e5b6abe3bdb/ Log: in-progress diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -16,9 +16,12 @@ CMD_BREAKPOINTS = 4 CMD_MOREINFO = 5 CMD_ATTACHID = 6 +CMD_WATCH = 7 +CMD_EXPECTED = 8 ANSWER_TEXT = 20 ANSWER_MOREINFO = 21 ANSWER_NEXTNID = 22 +ANSWER_COMPILED = 23 def stop_point(): @@ -33,9 +36,6 @@ def register_debug_command(command, lambda_func): """Register the extra RPython-implemented debug command.""" -def register_allocation_command(lambda_func): - """Register the extra RPython-implemented callback for allocation.""" - def send_answer(cmd, arg1=0, arg2=0, arg3=0, extra=""): """For RPython debug commands: writes an answer block to stdout""" llop.revdb_send_answer(lltype.Void, cmd, arg1, arg2, arg3, extra) @@ -126,55 +126,38 @@ _about_ = register_debug_command def compute_result_annotation(self, s_command_num, s_lambda_func): - from rpython.annotator import model as annmodel - from rpython.rtyper import llannotation - command_num = s_command_num.const lambda_func = s_lambda_func.const - assert isinstance(command_num, int) + assert isinstance(command_num, (int, str)) t = self.bookkeeper.annotator.translator if t.config.translation.reverse_debugger: func = lambda_func() try: cmds = t.revdb_commands except AttributeError: - cmds = t.revdb_commands = [] - for old_num, old_func in cmds: - if old_num == command_num: - assert old_func is func - break - else: - cmds.append((command_num, func)) + cmds = t.revdb_commands = {} + old_func = cmds.setdefault(command_num, func) + assert old_func is func s_func = self.bookkeeper.immutablevalue(func) - s_ptr1 = llannotation.SomePtr(ll_ptrtype=_CMDPTR) - s_str2 = annmodel.SomeString() + arg_getter = getattr(self, 'arguments_' + str(command_num), + self.default_arguments) self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key, - s_func, [s_ptr1, s_str2]) + s_func, arg_getter()) + + def default_arguments(self): + from rpython.annotator import model as annmodel + from rpython.rtyper import llannotation + return [llannotation.SomePtr(ll_ptrtype=_CMDPTR), + annmodel.SomeString()] + + def arguments_ALLOCATING(self): + from rpython.annotator import model as annmodel + from rpython.rtyper import llannotation + return [annmodel.SomeInteger(knowntype=r_longlong), + llannotation.lltype_to_annotation(llmemory.GCREF)] + + def arguments_WATCHING(self): + return [] def specialize_call(self, hop): hop.exception_cannot_occur() - - -class RegisterAllocationCommand(ExtRegistryEntry): - _about_ = register_allocation_command - - def compute_result_annotation(self, s_lambda_func): - from rpython.annotator import model as annmodel - from rpython.rtyper import llannotation - - lambda_func = s_lambda_func.const - t = self.bookkeeper.annotator.translator - if t.config.translation.reverse_debugger: - func = lambda_func() - try: - assert t.revdb_allocation_cmd is func - except AttributeError: - t.revdb_allocation_cmd = func - s_func = self.bookkeeper.immutablevalue(func) - s_int1 = annmodel.SomeInteger(knowntype=r_longlong) - s_ref2 = llannotation.lltype_to_annotation(llmemory.GCREF) - self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key, - s_func, [s_int1, s_ref2]) - - def specialize_call(self, hop): - hop.exception_cannot_occur() diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -27,27 +27,35 @@ lltype.Void)) ALLOCFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.LONGLONG, llmemory.GCREF], lltype.Void)) + WATCHFUNCPTR = lltype.Ptr(lltype.FuncType([], lltype.Signed)) bk = db.translator.annotator.bookkeeper - cmds = getattr(db.translator, 'revdb_commands', []) + cmds = getattr(db.translator, 'revdb_commands', {}) + numcmds = [(num, func) for (num, func) in cmds.items() + if isinstance(num, int)] S = lltype.Struct('RPY_REVDB_COMMANDS', - ('names', lltype.FixedSizeArray(rffi.INT, len(cmds) + 1)), - ('funcs', lltype.FixedSizeArray(FUNCPTR, len(cmds))), - ('alloc', ALLOCFUNCPTR)) + ('names', lltype.FixedSizeArray(rffi.INT, len(numcmds) + 1)), + ('funcs', lltype.FixedSizeArray(FUNCPTR, len(numcmds))), + ('alloc', ALLOCFUNCPTR), + ('watch', WATCHFUNCPTR)) s = lltype.malloc(S, flavor='raw', immortal=True, zero=True) - for i, (name, func) in enumerate(cmds): + i = 0 + for name, func in cmds.items(): fnptr = lltype.getfunctionptr(bk.getdesc(func).getuniquegraph()) - assert lltype.typeOf(fnptr) == FUNCPTR - assert isinstance(name, int) and name != 0 - s.names[i] = rffi.cast(rffi.INT, name) - s.funcs[i] = fnptr - - allocation_cmd = getattr(db.translator, 'revdb_allocation_cmd', None) - if allocation_cmd is not None: - s.alloc = lltype.getfunctionptr( - bk.getdesc(allocation_cmd).getuniquegraph()) + if isinstance(name, int): + assert name != 0 + s.names[i] = rffi.cast(rffi.INT, name) + s.funcs[i] = fnptr + i += 1 + elif name == "ALLOCATING": + s.alloc = fnptr + elif name == "WATCHING": + s.watch = fnptr + else: + raise AssertionError("bad tag in register_debug_command(): %r" + % (name,)) exports.EXPORTS_obj2name[s._as_obj()] = 'rpy_revdb_commands' db.get(s) diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -109,6 +109,15 @@ lst = [str(n) for n in sorted(self.pgroup.paused)] print ', '.join(lst) + def cmd_info_breakpoints(self): + """List current breakpoints""" + lst = self.pgroup.all_breakpoints.num2name.items() + if lst: + for num, name in sorted(lst): + print '%8d: %s' % (num, name) + else: + print 'no breakpoints.' + def move_forward(self, steps): self.remove_tainting() try: From pypy.commits at gmail.com Wed Jun 22 06:17:57 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 22 Jun 2016 03:17:57 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: a testing mixin for making we_are_jitted() return random results at runtime. Message-ID: <576a65d5.6372c20a.3456f.fffff7ca@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85323:1ac481910937 Date: 2016-06-20 11:45 +0200 http://bitbucket.org/pypy/pypy/changeset/1ac481910937/ Log: a testing mixin for making we_are_jitted() return random results at runtime. this is useful to check whether the semantics in both paths is really the same (which it has to be). Apply this in some objspace tests. diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -1,6 +1,5 @@ """The builtin str implementation""" -from rpython.rlib.jit import we_are_jitted from rpython.rlib.objectmodel import ( compute_hash, compute_unique_id, import_from_mixin) from rpython.rlib.buffer import StringBuffer diff --git a/pypy/objspace/std/test/test_callmethod.py b/pypy/objspace/std/test/test_callmethod.py --- a/pypy/objspace/std/test/test_callmethod.py +++ b/pypy/objspace/std/test/test_callmethod.py @@ -1,4 +1,6 @@ -class AppTestCallMethod: +from rpython.rlib import jit + +class AppTestCallMethod(jit.RandomWeAreJittedTestMixin): # The exec hacking is needed to have the code snippets compiled # by our own compiler, not CPython's diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,6 +1,9 @@ import pytest from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictObject from pypy.objspace.std.mapdict import * +from rpython.rlib import jit +import random + class Config: class objspace: @@ -662,7 +665,7 @@ # XXX write more -class AppTestWithMapDict(object): +class AppTestWithMapDict(jit.RandomWeAreJittedTestMixin): def test_simple(self): class A(object): diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef +from rpython.rlib import jit class TestTypeObject: @@ -59,7 +60,7 @@ assert len(warnings) == 2 -class AppTestTypeObject: +class AppTestTypeObject(jit.RandomWeAreJittedTestMixin): def test_abstract_methods(self): class X(object): pass diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -378,17 +378,38 @@ hop.exception_cannot_occur() return hop.genop('hint', [v, c_hint], resulttype=v.concretetype) +def _we_are_jitted_interpreted(): + return False # for monkey-patching def we_are_jitted(): """ Considered as true during tracing and blackholing, so its consquences are reflected into jitted code """ # during testing we return something randomly, to emulate the real # behaviour where you can switch to tracing a arbitrary points. - return False + return _we_are_jitted_interpreted() _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */', default=0) + +class RandomWeAreJittedTestMixin(object): + def setup_method(self, meth): + global _we_are_jitted_interpreted + seed = random.random() + print "seed", seed + random.seed(seed) + self.orig_we_are_jitted = _we_are_jitted_interpreted + def _we_are_jitted_interpreted_random(): + result = random.random() > 0.5 + return result + _we_are_jitted_interpreted = _we_are_jitted_interpreted_random + + def teardown_method(self, meth): + global _we_are_jitted_interpreted + _we_are_jitted_interpreted = self.orig_we_are_jitted + + + def _get_virtualizable_token(frame): """ An obscure API to get vable token. Used by _vmprof diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -5,7 +5,8 @@ from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote, JitHintError, oopspec, isconstant, conditional_call, elidable, unroll_safe, dont_look_inside, - enter_portal_frame, leave_portal_frame, elidable_compatible) + enter_portal_frame, leave_portal_frame, elidable_compatible, + RandomWeAreJittedTestMixin) from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rtyper.lltypesystem import lltype @@ -335,3 +336,19 @@ leave_portal_frame() t = Translation(g, []) t.compile_c() # does not crash + +class Test_we_are_jitted(RandomWeAreJittedTestMixin): + def test_bad_we_are_jitted(self): + def bad(x): + if we_are_jitted(): + return x + 1 + else: + return x + 2 + + try: + for i in range(100): + assert bad(1) == 3 + except AssertionError: + pass + else: + assert 0, "should have failed" From pypy.commits at gmail.com Wed Jun 22 06:18:01 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 22 Jun 2016 03:18:01 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fix an error message Message-ID: <576a65d9.c91d1c0a.86532.ffffa738@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85325:1a4c8be91191 Date: 2016-06-20 15:54 +0200 http://bitbucket.org/pypy/pypy/changeset/1a4c8be91191/ Log: fix an error message diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -228,7 +228,7 @@ if not ccond.known_valid.same_constant(op.getarg(1)): r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop( op) - raise InvalidLoop('A GUARD_VALUE (%s) ' + raise InvalidLoop('A GUARD_COMPATIBLE (%s) ' 'was proven to always fail' % r) return else: From pypy.commits at gmail.com Wed Jun 22 06:18:02 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 22 Jun 2016 03:18:02 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: an XXX Message-ID: <576a65da.9a4a1c0a.8f94e.ffffa484@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85326:3c8821d370d8 Date: 2016-06-22 12:16 +0200 http://bitbucket.org/pypy/pypy/changeset/3c8821d370d8/ Log: an XXX diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -230,6 +230,8 @@ op) raise InvalidLoop('A GUARD_COMPATIBLE (%s) ' 'was proven to always fail' % r) + # XXX check that the runtime constant matches the previous set of + # conditions too return else: info._compatibility_conditions = CompatibilityCondition( From pypy.commits at gmail.com Wed Jun 22 06:18:04 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 22 Jun 2016 03:18:04 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: pass the first unroll test (way too easy???) Message-ID: <576a65dc.c5301c0a.955eb.ffffbb6a@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85327:b5babc443dd9 Date: 2016-06-22 12:17 +0200 http://bitbucket.org/pypy/pypy/changeset/b5babc443dd9/ Log: pass the first unroll test (way too easy???) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py --- a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py @@ -3,6 +3,8 @@ LLtypeMixin) from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import ( BaseTestBasic) +from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import ( + BaseTestWithUnroll) from rpython.jit.metainterp.history import ConstInt, ConstPtr from rpython.jit.metainterp.optimize import InvalidLoop @@ -278,3 +280,25 @@ (ConstInt(123), ConstPtr(self.quasiptr), ConstInt(-4247)): ConstInt(5), } self.optimize_loop(ops, expected, call_pure_results) + + +class TestCompatibleUnroll(BaseTestWithUnroll, LLtypeMixin): + + def test_remove_guard_compatible(self): + ops = """ + [p0] + guard_compatible(p0, ConstPtr(myptr)) [] + guard_compatible(p0, ConstPtr(myptr)) [] + jump(p0) + """ + preamble = """ + [p0] + guard_compatible(p0, ConstPtr(myptr)) [] + jump(p0) + """ + expected = """ + [p0] + jump(p0) + """ + self.optimize_loop(ops, expected, expected_preamble=preamble) + diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -85,6 +85,10 @@ op.set_forwarded(str_info) if preamble_info.is_nonnull(): self.make_nonnull(op) + if preamble_info._compatibility_conditions: + info_in_loop = op.get_forwarded() + if info_in_loop is not None: + info_in_loop._compatibility_conditions = preamble_info._compatibility_conditions elif isinstance(preamble_info, intutils.IntBound): if preamble_info.lower > MININT/2 or preamble_info.upper < MAXINT/2: intbound = self.getintbound(op) From pypy.commits at gmail.com Wed Jun 22 06:17:59 2016 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 22 Jun 2016 03:17:59 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: merge default Message-ID: <576a65d7.4aa71c0a.3bd5d.ffffaa4b@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85324:914cc74b5a8c Date: 2016-06-20 15:49 +0200 http://bitbucket.org/pypy/pypy/changeset/914cc74b5a8c/ Log: merge default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -26,3 +26,4 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 +7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.9 +Version: 0.4.10 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.9" +__version__ = "0.4.10" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -315,13 +315,28 @@ - ``complex`` + - ``str`` (empty or single-character strings only) + + - ``unicode`` (empty or single-character strings only) + + - ``tuple`` (empty tuples only) + + - ``frozenset`` (empty frozenset only) + This change requires some changes to ``id`` as well. ``id`` fulfills the following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the above types will return a value that is computed from the argument, and can thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long). -Notably missing from the list above are ``str`` and ``unicode``. If your -code relies on comparing strings with ``is``, then it might break in PyPy. +Note that strings of length 2 or greater can be equal without being +identical. Similarly, ``x is (2,)`` is not necessarily true even if +``x`` contains a tuple and ``x == (2,)``. The uniqueness rules apply +only to the particular cases described above. The ``str``, ``unicode``, +``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a +test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was +equal to ``"?"`` or ``()``. The new behavior added in PyPy 5.4 is +closer to CPython's, which caches precisely the empty tuple/frozenset, +and (generally but not always) the strings and unicodes of length <= 1. Note that for floats there "``is``" only one object per "bit pattern" of the float. So ``float('nan') is float('nan')`` is true on PyPy, diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.1.rst whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.1.rst @@ -0,0 +1,41 @@ +========== +PyPy 5.3.1 +========== + +We have released a bugfix for PyPy2.7-v5.3.0, released last week, +due to issues_ reported by users. + +Thanks to those who reported the issues. + +.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,10 @@ .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 +.. 418b05f95db5 +Improve CPython compatibility for ``is``. Now code like ``if x is ():`` +works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id . + .. pull request #455 Add sys.{get,set}dlopenflags, for cpyext extensions. @@ -31,3 +35,5 @@ Simplify handling of interp-level tests and make it more forward- compatible. +.. branch: pyfile-tell +Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst @@ -0,0 +1,15 @@ +=========================== +What's new in PyPy2.7 5.3.1 +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.3.0 +.. startrev: f4d726d1a010 + + +A bug-fix release, merging these changes: + + * Add include guards to pymem.h, fixes issue #2321 + + * Make vmprof build on OpenBSD, from pull request #456 + + * Fix ``bytearray('').replace('a', 'ab')``, issue #2324 diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.3.1-alpha0" -#define PYPY_VERSION_NUM 0x05030100 +#define PYPY_VERSION "5.3.2-alpha0" +#define PYPY_VERSION_NUM 0x05030200 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -55,6 +55,7 @@ if not PyFile_Check(space, w_p): raise oefmt(space.w_IOError, 'first argument must be an open file') assert isinstance(w_p, W_File) + w_p.stream.flush_buffers() try: fd = space.int_w(space.call_method(w_p, 'fileno')) mode = w_p.mode diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -43,15 +43,6 @@ /* foo methods */ -static void -foo_dealloc(fooobject *foop) -{ - PyObject_Del(foop); -} - - -/* foo methods-as-attributes */ - static PyObject * foo_copy(fooobject *self) { @@ -195,7 +186,7 @@ sizeof(fooobject), /*tp_size*/ 0, /*tp_itemsize*/ /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -463,9 +454,9 @@ PyTypeObject InitErrType = { PyObject_HEAD_INIT(NULL) 0, - "foo.InitErr", - sizeof(PyObject), - 0, + "foo.InitErrType", + sizeof(PyObject),/*tp_basicsize*/ + 0, /*tp_itemsize*/ 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ @@ -508,12 +499,12 @@ 0, /*tp_dictoffset*/ initerrtype_init, /*tp_init*/ - 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_alloc*/ 0, /*tp_new*/ - 0, /*tp_free Low-level free-memory routine */ - 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ 0, /*tp_bases*/ - 0, /*tp_mro method resolution order */ + 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0 /*tp_weaklist*/ diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -1,5 +1,7 @@ +from pypy.conftest import option from pypy.module.cpyext.api import fopen, fclose, fwrite from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.object import Py_PRINT_RAW from rpython.rtyper.lltypesystem import rffi, lltype from rpython.tool.udir import udir @@ -111,3 +113,34 @@ out, err = capfd.readouterr() out = out.replace('\r\n', '\n') assert out == " 1 23\n" + + +class AppTestPyFile(AppTestCpythonExtensionBase): + + def setup_class(cls): + from rpython.tool.udir import udir + if option.runappdirect: + cls.w_udir = str(udir) + else: + cls.w_udir = cls.space.wrap(str(udir)) + + def test_file_tell(self): + module = self.import_extension('foo', [ + ("get_c_tell", "METH_O", + """ + FILE * fp = PyFile_AsFile(args); + if (fp == NULL) + return PyLong_FromLong(0); + return PyLong_FromLong(ftell(fp)); + """), + ]) + filename = self.udir + "/_test_file" + with open(filename, 'w') as fid: + fid.write('3' * 122) + with open(filename, 'r') as fid: + s = fid.read(80) + t_py = fid.tell() + assert t_py == 80 + t_c = module.get_c_tell(fid) + assert t_c == t_py + diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -142,6 +142,7 @@ ref = rffi.cast(PyTupleObject, ref) size = ref.c_ob_size if index < 0 or index >= size: + decref(space, py_obj) raise oefmt(space.w_IndexError, "tuple assignment index out of range") old_ref = ref.c_ob_item[index] ref.c_ob_item[index] = py_obj # consumes a reference diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py --- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py +++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py @@ -163,14 +163,10 @@ guard_not_invalidated(descr=...) i32 = float_ne(f31, 0.000000) guard_true(i32, descr=...) - i34 = getarrayitem_raw_i(#, #, descr=) # XXX what are these? - guard_value(i34, #, descr=...) # XXX don't appear in - i35 = getarrayitem_raw_i(#, #, descr=) # XXX equiv test_zjit i36 = int_add(i24, 1) i37 = int_add(i29, 8) i38 = int_ge(i36, i30) guard_false(i38, descr=...) - guard_value(i35, #, descr=...) # XXX jump(..., descr=...) """) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 3, 1, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 2, "alpha", 0) #XXX # sync patchlevel.h import pypy diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -17,6 +17,7 @@ from pypy.objspace.std.unicodeobject import ( decode_object, unicode_from_encoded_object, unicode_from_string, getdefaultencoding) +from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT class W_AbstractBytesObject(W_Root): @@ -29,12 +30,26 @@ return True if self.user_overridden_class or w_other.user_overridden_class: return False - return space.str_w(self) is space.str_w(w_other) + s1 = space.str_w(self) + s2 = space.str_w(w_other) + if len(s2) > 1: + return s1 is s2 + else: # strings of len <= 1 are unique-ified + return s1 == s2 def immutable_unique_id(self, space): if self.user_overridden_class: return None - return space.wrap(compute_unique_id(space.str_w(self))) + s = space.str_w(self) + if len(s) > 1: + uid = compute_unique_id(s) + else: # strings of len <= 1 are unique-ified + if len(s) == 1: + base = ord(s[0]) # base values 0-255 + else: + base = 256 # empty string: base value 256 + uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) def unicode_w(self, space): # Use the default encoding. diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -6,6 +6,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.unicodeobject import W_UnicodeObject +from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT from rpython.rlib.objectmodel import r_dict from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash @@ -575,6 +576,23 @@ class W_FrozensetObject(W_BaseSetObject): hash = 0 + def is_w(self, space, w_other): + if not isinstance(w_other, W_FrozensetObject): + return False + if self is w_other: + return True + if self.user_overridden_class or w_other.user_overridden_class: + return False + # empty frozensets are unique-ified + return 0 == w_other.length() == self.length() + + def immutable_unique_id(self, space): + if self.user_overridden_class or self.length() > 0: + return None + # empty frozenset: base value 259 + uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) + def _newobj(self, space, w_iterable): """Make a new frozenset by taking ownership of 'w_iterable'.""" if type(self) is W_FrozensetObject: diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -162,7 +162,8 @@ buffer = _get_buffer(space, w_sub) res = count(value, buffer, start, end) - return space.wrap(max(res, 0)) + assert res >= 0 + return space.wrap(res) def descr_decode(self, space, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodeobject import ( diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -211,6 +211,7 @@ check(bytearray('abc').replace('b', bytearray('d')), 'adc') check(bytearray('abc').replace('b', 'd'), 'adc') + check(bytearray('').replace('a', 'ab'), '') check(bytearray('abc').upper(), 'ABC') check(bytearray('ABC').lower(), 'abc') diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -186,17 +186,36 @@ def test_id_on_strs(self): if self.appdirect: skip("cannot run this test as apptest") - u = u"a" - assert id(self.unwrap_wrap_unicode(u)) == id(u) - s = "a" - assert id(self.unwrap_wrap_str(s)) == id(s) + for u in [u"", u"a", u"aa"]: + assert id(self.unwrap_wrap_unicode(u)) == id(u) + s = str(u) + assert id(self.unwrap_wrap_str(s)) == id(s) + # + assert id('') == (256 << 4) | 11 # always + assert id(u'') == (257 << 4) | 11 + assert id('a') == (ord('a') << 4) | 11 + assert id(u'\u1234') == ((~0x1234) << 4) | 11 + + def test_id_of_tuples(self): + l = [] + x = (l,) + assert id(x) != id((l,)) # no caching at all + if self.appdirect: + skip("cannot run this test as apptest") + assert id(()) == (258 << 4) | 11 # always + + def test_id_of_frozensets(self): + x = frozenset([4]) + assert id(x) != id(frozenset([4])) # no caching at all + if self.appdirect: + skip("cannot run this test as apptest") + assert id(frozenset()) == (259 << 4) | 11 # always + assert id(frozenset([])) == (259 << 4) | 11 # always def test_identity_vs_id_primitives(self): - if self.cpython_apptest: - skip("cpython behaves differently") import sys - l = range(-10, 10) - for i in range(10): + l = range(-10, 10, 2) + for i in [0, 1, 3]: l.append(float(i)) l.append(i + 0.1) l.append(long(i)) @@ -206,18 +225,15 @@ l.append(i - 1j) l.append(1 + i * 1j) l.append(1 - i * 1j) - s = str(i) - l.append(s) - u = unicode(s) - l.append(u) + l.append((i,)) + l.append(frozenset([i])) l.append(-0.0) l.append(None) l.append(True) l.append(False) - s = "s" - l.append(s) - s = u"s" - l.append(s) + l.append(()) + l.append(tuple([])) + l.append(frozenset()) for i, a in enumerate(l): for b in l[i:]: @@ -228,21 +244,18 @@ def test_identity_vs_id_str(self): if self.appdirect: skip("cannot run this test as apptest") - import sys - l = range(-10, 10) - for i in range(10): - s = str(i) + l = [] + def add(s, u): l.append(s) l.append(self.unwrap_wrap_str(s)) - u = unicode(s) + l.append(s[:1] + s[1:]) l.append(u) l.append(self.unwrap_wrap_unicode(u)) - s = "s" - l.append(s) - l.append(self.unwrap_wrap_str(s)) - s = u"s" - l.append(s) - l.append(self.unwrap_wrap_unicode(s)) + l.append(u[:1] + u[1:]) + for i in range(3, 18): + add(str(i), unicode(i)) + add("s", u"s") + add("", u"") for i, a in enumerate(l): for b in l[i:]: diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -9,7 +9,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.objspace.std.sliceobject import (W_SliceObject, unwrap_start_stop, normalize_simple_slice) -from pypy.objspace.std.util import negate +from pypy.objspace.std.util import negate, IDTAG_SPECIAL, IDTAG_SHIFT from rpython.rlib import jit from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.rarithmetic import intmask @@ -38,6 +38,23 @@ class W_AbstractTupleObject(W_Root): __slots__ = () + def is_w(self, space, w_other): + if not isinstance(w_other, W_AbstractTupleObject): + return False + if self is w_other: + return True + if self.user_overridden_class or w_other.user_overridden_class: + return False + # empty tuples are unique-ified + return 0 == w_other.length() == self.length() + + def immutable_unique_id(self, space): + if self.user_overridden_class or self.length() > 0: + return None + # empty tuple: base value 258 + uid = (258 << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) + def __repr__(self): """representation for debugging purposes""" reprlist = [repr(w_item) for w_item in self.tolist()] diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -18,6 +18,7 @@ from pypy.objspace.std.basestringtype import basestring_typedef from pypy.objspace.std.formatting import mod_format from pypy.objspace.std.stringmethods import StringMethods +from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode', 'encode_object', 'decode_object', 'unicode_from_object', @@ -52,12 +53,26 @@ return True if self.user_overridden_class or w_other.user_overridden_class: return False - return space.unicode_w(self) is space.unicode_w(w_other) + s1 = space.unicode_w(self) + s2 = space.unicode_w(w_other) + if len(s2) > 1: + return s1 is s2 + else: # strings of len <= 1 are unique-ified + return s1 == s2 def immutable_unique_id(self, space): if self.user_overridden_class: return None - return space.wrap(compute_unique_id(space.unicode_w(self))) + s = space.unicode_w(self) + if len(s) > 1: + uid = compute_unique_id(s) + else: # strings of len <= 1 are unique-ified + if len(s) == 1: + base = ~ord(s[0]) # negative base values + else: + base = 257 # empty unicode string: base value 257 + uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) def str_w(self, space): return space.str_w(space.str(self)) diff --git a/pypy/objspace/std/util.py b/pypy/objspace/std/util.py --- a/pypy/objspace/std/util.py +++ b/pypy/objspace/std/util.py @@ -9,6 +9,12 @@ IDTAG_FLOAT = 5 IDTAG_COMPLEX = 7 IDTAG_METHOD = 9 +IDTAG_SPECIAL = 11 # -1 - (-maxunicode-1): unichar + # 0 - 255: char + # 256: empty string + # 257: empty unicode + # 258: empty tuple + # 259: empty frozenset CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=') BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>', diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,9 +1,9 @@ # Edit these appropriately before running this script maj=5 min=3 -rev=0 +rev=1 branchname=release-$maj.x # ==OR== release-$maj.$min.x -tagname=release-pypy2.7-v$maj.$min # ==OR== release-$maj.$min +tagname=release-pypy2.7-v$maj.$min.$rev # ==OR== release-$maj.$min echo checking hg log -r $branchname hg log -r $branchname || exit 1 diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -2,7 +2,7 @@ import py from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\ VStructStateInfo, LEVEL_CONSTANT,\ - VArrayStateInfo, NotVirtualStateInfo, VirtualState,\ + VArrayStateInfo, not_virtual, VirtualState,\ GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\ @@ -31,10 +31,10 @@ def setup_class(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - self.knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + self.knownclass_info = not_virtual(self.cpu, 'r', value) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - self.knownclass_info2 = NotVirtualStateInfo(self.cpu, 'r', value) + self.knownclass_info2 = not_virtual(self.cpu, 'r', value) def guards(self, info1, info2, box, runtime_box, expected, inputargs=None): if inputargs is None: @@ -80,7 +80,7 @@ def test_make_inputargs(self): optimizer = FakeOptimizer(self.cpu) args = [InputArgInt()] - info0 = NotVirtualStateInfo(self.cpu, args[0].type, None) + info0 = not_virtual(self.cpu, args[0].type, None) vs = VirtualState([info0]) assert vs.make_inputargs(args, optimizer) == args info0.level = LEVEL_CONSTANT @@ -108,8 +108,8 @@ assert info1 in state.bad and info2 in state.bad for BoxType in (InputArgInt, InputArgFloat, InputArgRef): - info1 = NotVirtualStateInfo(self.cpu, BoxType.type, None) - info2 = NotVirtualStateInfo(self.cpu, BoxType.type, None) + info1 = not_virtual(self.cpu, BoxType.type, None) + info2 = not_virtual(self.cpu, BoxType.type, None) postest(info1, info2) info1, info2 = VArrayStateInfo(42), VArrayStateInfo(42) @@ -126,9 +126,9 @@ def test_NotVirtualStateInfo_generalization(self): def isgeneral(tp1, info1, tp2, info2): - info1 = NotVirtualStateInfo(self.cpu, tp1, info1) + info1 = not_virtual(self.cpu, tp1, info1) info1.position = 0 - info2 = NotVirtualStateInfo(self.cpu, tp2, info2) + info2 = not_virtual(self.cpu, tp2, info2) info2.position = 0 return VirtualState([info1]).generalization_of(VirtualState([info2]), FakeOptimizer(self.cpu)) @@ -166,8 +166,8 @@ assert not isgeneral('r', value1, 'r', value2) def test_field_matching_generalization(self): - const1 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(1)) - const2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(2)) + const1 = not_virtual(self.cpu, 'i', ConstIntBound(1)) + const2 = not_virtual(self.cpu, 'i', ConstIntBound(2)) const1.position = const2.position = 1 self.check_invalid(const1, const2) self.check_invalid(const2, const1) @@ -192,16 +192,16 @@ def test_known_class_generalization(self): knownclass1 = info.InstancePtrInfo(None, ConstPtr(self.myptr)) - info1 = NotVirtualStateInfo(self.cpu, 'r', knownclass1) + info1 = not_virtual(self.cpu, 'r', knownclass1) info1.position = 0 knownclass2 = info.InstancePtrInfo(None, ConstPtr(self.myptr)) - info2 = NotVirtualStateInfo(self.cpu, 'r', knownclass2) + info2 = not_virtual(self.cpu, 'r', knownclass2) info2.position = 0 self.check_no_guards(info1, info2) self.check_no_guards(info2, info1) knownclass3 = info.InstancePtrInfo(None, ConstPtr(self.myptr2)) - info3 = NotVirtualStateInfo(self.cpu, 'r', knownclass3) + info3 = not_virtual(self.cpu, 'r', knownclass3) info3.position = 0 self.check_invalid(info1, info3) self.check_invalid(info2, info3) @@ -222,26 +222,26 @@ #unknown_val = PtrOptValue(self.nodebox) #unknownnull_val = PtrOptValue(BoxPtr(self.nullptr)) opt = FakeOptimizer(self.cpu) - unknown_info = NotVirtualStateInfo(self.cpu, 'r', None) + unknown_info = not_virtual(self.cpu, 'r', None) - nonnull_info = NotVirtualStateInfo(self.cpu, 'r', info.NonNullPtrInfo()) + nonnull_info = not_virtual(self.cpu, 'r', info.NonNullPtrInfo()) classbox1 = self.cpu.ts.cls_of_box(ConstPtr(self.nodeaddr)) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', - info.InstancePtrInfo(None, classbox1)) + knownclass_info = not_virtual(self.cpu, 'r', + info.InstancePtrInfo(None, classbox1)) classbox2 = self.cpu.ts.cls_of_box(ConstPtr(self.node2addr)) - knownclass2_info = NotVirtualStateInfo(self.cpu, 'r', - info.InstancePtrInfo(None, classbox2)) + knownclass2_info = not_virtual(self.cpu, 'r', + info.InstancePtrInfo(None, classbox2)) - constant_info = NotVirtualStateInfo(self.cpu, 'i', - ConstIntBound(1)) - constant_ptr_info = NotVirtualStateInfo(self.cpu, 'r', + constant_info = not_virtual(self.cpu, 'i', + ConstIntBound(1)) + constant_ptr_info = not_virtual(self.cpu, 'r', info.ConstPtrInfo(ConstPtr(self.nodeaddr))) constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr)) - constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val) - constclass2_info = NotVirtualStateInfo(self.cpu, 'r', + constclass_info = not_virtual(self.cpu, 'r', constclass_val) + constclass2_info = not_virtual(self.cpu, 'r', info.ConstPtrInfo(ConstPtr(self.node2addr))) - constantnull_info = NotVirtualStateInfo(self.cpu, 'r', + constantnull_info = not_virtual(self.cpu, 'r', info.ConstPtrInfo(ConstPtr(self.nullptr))) # unknown unknown @@ -260,9 +260,10 @@ self.check_no_guards(unknown_info, knownclass_info) # unknown constant - self.check_no_guards(unknown_info, constant_info, + unknown_info_int = not_virtual(self.cpu, 'i', None) + self.check_no_guards(unknown_info_int, constant_info, ConstInt(1), ConstIntBound(1)) - self.check_no_guards(unknown_info, constant_info) + self.check_no_guards(unknown_info_int, constant_info) # nonnull unknown @@ -293,11 +294,11 @@ const_nonnull = ConstPtr(self.nodeaddr) const_nonnull2 = ConstPtr(self.node2addr) const_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO)) - self.check_no_guards(nonnull_info, constant_info, const_nonnull, + self.check_no_guards(nonnull_info, constant_ptr_info, const_nonnull, info.ConstPtrInfo(const_nonnull)) self.check_invalid(nonnull_info, constantnull_info, const_null, info.ConstPtrInfo(const_null)) - self.check_no_guards(nonnull_info, constant_info) + self.check_no_guards(nonnull_info, constant_ptr_info) self.check_invalid(nonnull_info, constantnull_info) @@ -392,8 +393,8 @@ value1 = IntUnbounded() value1.make_ge(IntBound(0, 10)) value1.make_le(IntBound(20, 30)) - info1 = NotVirtualStateInfo(self.cpu, 'i', value1) - info2 = NotVirtualStateInfo(self.cpu, 'i', IntUnbounded()) + info1 = not_virtual(self.cpu, 'i', value1) + info2 = not_virtual(self.cpu, 'i', IntUnbounded()) expected = """ [i0] i1 = int_ge(i0, 0) @@ -408,18 +409,18 @@ value1 = IntUnbounded() value1.make_ge(IntBound(0, 10)) value1.make_le(IntBound(20, 30)) - info1 = NotVirtualStateInfo(self.cpu, 'i', value1) - info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(10000)) + info1 = not_virtual(self.cpu, 'i', value1) + info2 = not_virtual(self.cpu, 'i', ConstIntBound(10000)) self.check_invalid(info1, info2) - info1 = NotVirtualStateInfo(self.cpu, 'i', value1) - info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(11)) + info1 = not_virtual(self.cpu, 'i', value1) + info2 = not_virtual(self.cpu, 'i', ConstIntBound(11)) self.check_no_guards(info1, info2) def test_known_class(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value1 = info.InstancePtrInfo(None, classbox) - info1 = NotVirtualStateInfo(self.cpu, 'r', value1) - info2 = NotVirtualStateInfo(self.cpu, 'r', None) + info1 = not_virtual(self.cpu, 'r', value1) + info2 = not_virtual(self.cpu, 'r', None) expected = """ [p0] guard_nonnull_class(p0, ConstClass(node_vtable)) [] @@ -456,18 +457,18 @@ def test_equal_inputargs(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) vstate1 = VirtualState([knownclass_info, knownclass_info]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) - unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None) + unknown_info1 = not_virtual(self.cpu, 'r', None) vstate2 = VirtualState([unknown_info1, unknown_info1]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu)) assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu)) - unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None) - unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', None) + unknown_info1 = not_virtual(self.cpu, 'r', None) + unknown_info2 = not_virtual(self.cpu, 'r', None) vstate3 = VirtualState([unknown_info1, unknown_info2]) assert vstate3.generalization_of(vstate2, FakeOptimizer(self.cpu)) assert vstate3.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -494,9 +495,9 @@ def test_generate_guards_on_virtual_fields_matches_array(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) innervalue1 = info.InstancePtrInfo(None, classbox) - innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1) + innerinfo1 = not_virtual(self.cpu, 'r', innervalue1) innerinfo1.position = 1 - innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None) + innerinfo2 = not_virtual(self.cpu, 'r', None) innerinfo2.position = 1 descr = ArrayDescr(lltype.GcArray(llmemory.GCREF), self.cpu) @@ -524,9 +525,9 @@ def test_generate_guards_on_virtual_fields_matches_instance(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) innervalue1 = info.InstancePtrInfo(None, classbox) - innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1) + innerinfo1 = not_virtual(self.cpu, 'r', innervalue1) innerinfo1.position = 1 - innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None) + innerinfo2 = not_virtual(self.cpu, 'r', None) innerinfo2.position = 1 info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr]) @@ -552,9 +553,9 @@ def test_generate_guards_on_virtual_fields_matches_struct(self): constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) innervalue1 = info.InstancePtrInfo(None, constclassbox) - innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1) + innerinfo1 = not_virtual(self.cpu, 'r', innervalue1) innerinfo1.position = 1 - innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None) + innerinfo2 = not_virtual(self.cpu, 'r', None) innerinfo2.position = 1 structdescr = self.nodesize @@ -583,9 +584,9 @@ def test_generate_guards_on_virtual_fields_matches_arraystruct(self): constclassbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) innervalue1 = info.InstancePtrInfo(None, constclassbox) - innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1) + innerinfo1 = not_virtual(self.cpu, 'r', innervalue1) innerinfo1.position = 1 - innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None) + innerinfo2 = not_virtual(self.cpu, 'r', None) innerinfo2.position = 1 NODE = lltype.Struct('NODE', ('x', llmemory.GCREF)) @@ -627,7 +628,7 @@ assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) info2 = VirtualStateInfo(ConstInt(42), [1, 2]) - unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', + unknown_info1 = not_virtual(self.cpu, 'r', info.InstancePtrInfo()) info2.fieldstate = [unknown_info1, unknown_info1] vstate2 = VirtualState([info2]) @@ -636,9 +637,9 @@ assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu)) info3 = VirtualStateInfo(ConstInt(42), [1, 2]) - unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', + unknown_info1 = not_virtual(self.cpu, 'r', info.InstancePtrInfo()) - unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', + unknown_info2 = not_virtual(self.cpu, 'r', info.InstancePtrInfo()) info3.fieldstate = [unknown_info1, unknown_info2] vstate3 = VirtualState([info3]) @@ -651,7 +652,7 @@ info1 = VirtualStateInfo(ConstInt(42), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -659,7 +660,7 @@ info2 = VirtualStateInfo(ConstInt(42), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) @@ -671,7 +672,7 @@ info1 = VirtualStateInfo(ConstInt(42), [10, 20]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -679,7 +680,7 @@ info2 = VirtualStateInfo(ConstInt(42), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) @@ -691,7 +692,7 @@ info1 = VirtualStateInfo(ConstInt(42), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -699,24 +700,24 @@ info2 = VirtualStateInfo(ConstInt(7), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) assert not vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu)) assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu)) - + def test_nonvirtual_is_not_virtual(self): info1 = VirtualStateInfo(ConstInt(42), [1, 2]) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) - info2 = NotVirtualStateInfo(self.cpu, 'r', value) + info2 = not_virtual(self.cpu, 'r', value) vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) @@ -727,7 +728,7 @@ info1 = VArrayStateInfo(42) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -735,7 +736,7 @@ info2 = VArrayStateInfo(42) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info2.fieldstate = [knownclass_info, knownclass_info] vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) @@ -747,7 +748,7 @@ info1 = VArrayStateInfo(42) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info1.fieldstate = [knownclass_info, knownclass_info] vstate1 = VirtualState([info1]) assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu)) @@ -755,7 +756,7 @@ info2 = VArrayStateInfo(42) classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr)) value = info.InstancePtrInfo(None, classbox) - knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value) + knownclass_info = not_virtual(self.cpu, 'r', value) info2.fieldstate = [knownclass_info] vstate2 = VirtualState([info2]) assert vstate2.generalization_of(vstate2, FakeOptimizer(self.cpu)) @@ -793,7 +794,7 @@ def test_crash_varay_clear(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) innervalue1 = info.InstancePtrInfo(None, classbox) - innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1) + innerinfo1 = not_virtual(self.cpu, 'r', innervalue1) innerinfo1.position = 1 innerinfo1.position_in_notvirtuals = 0 diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -350,40 +350,23 @@ def debug_header(self, indent): debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position) + +def not_virtual(cpu, type, info): + if type == 'i': + return NotVirtualStateInfoInt(cpu, type, info) + if type == 'r': + return NotVirtualStateInfoPtr(cpu, type, info) + return NotVirtualStateInfo(cpu, type, info) + + class NotVirtualStateInfo(AbstractVirtualStateInfo): - lenbound = None - intbound = None level = LEVEL_UNKNOWN constbox = None - known_class = None - + def __init__(self, cpu, type, info): if info and info.is_constant(): self.level = LEVEL_CONSTANT self.constbox = info.getconst() - if type == 'r': - self.known_class = info.get_known_class(cpu) - elif type == 'i': - self.intbound = info - elif type == 'r': - if info: - self.known_class = info.get_known_class(cpu) - if self.known_class: - self.level = LEVEL_KNOWNCLASS - elif info.is_nonnull(): - self.level = LEVEL_NONNULL - self.lenbound = info.getlenbound(None) - elif type == 'i': - if isinstance(info, IntBound): - if info.lower < MININT / 2: - info.lower = MININT - if info.upper > MAXINT / 2: - info.upper = MAXINT - self.intbound = info - elif type == 'f': - if info and info.is_constant(): - self.level = LEVEL_CONSTANT - self.constbox = info.getconst() def is_const(self): return self.constbox is not None @@ -394,84 +377,15 @@ def _generate_guards(self, other, box, runtime_box, state): # XXX This will always retrace instead of forcing anything which # might be what we want sometimes? - if not isinstance(other, NotVirtualStateInfo): - raise VirtualStatesCantMatch( - 'The VirtualStates does not match as a ' + - 'virtual appears where a pointer is needed ' + - 'and it is too late to force it.') - - extra_guards = state.extra_guards - cpu = state.cpu - if self.lenbound: - if other.lenbound is None: - other_bound = IntLowerBound(0) - else: - other_bound = other.lenbound - if not self.lenbound.contains_bound(other_bound): - raise VirtualStatesCantMatch("length bound does not match") - if self.level == LEVEL_UNKNOWN: - # confusingly enough, this is done also for pointers - # which have the full range as the "bound", so it always works - return self._generate_guards_intbounds(other, box, runtime_box, - extra_guards, - state.optimizer) - - # the following conditions often peek into the runtime value that the - # box had when tracing. This value is only used as an educated guess. - # It is used here to choose between either emitting a guard and jumping - # to an existing compiled loop or retracing the loop. Both alternatives - # will always generate correct behaviour, but performance will differ. - elif self.level == LEVEL_NONNULL: - if other.level == LEVEL_UNKNOWN: - if runtime_box is not None and runtime_box.nonnull(): - op = ResOperation(rop.GUARD_NONNULL, [box]) - extra_guards.append(op) - return - else: - raise VirtualStatesCantMatch("other not known to be nonnull") - elif other.level == LEVEL_NONNULL: - return - elif other.level == LEVEL_KNOWNCLASS: - return # implies nonnull - else: - assert other.level == LEVEL_CONSTANT - assert other.constbox - if not other.constbox.nonnull(): - raise VirtualStatesCantMatch("constant is null") - return - - elif self.level == LEVEL_KNOWNCLASS: - if other.level == LEVEL_UNKNOWN: - if (runtime_box and runtime_box.nonnull() and - self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))): - op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class]) - extra_guards.append(op) - return - else: - raise VirtualStatesCantMatch("other's class is unknown") - elif other.level == LEVEL_NONNULL: - if (runtime_box and self.known_class.same_constant( - cpu.ts.cls_of_box(runtime_box))): - op = ResOperation(rop.GUARD_CLASS, [box, self.known_class]) - extra_guards.append(op) - return - else: - raise VirtualStatesCantMatch("other's class is unknown") - elif other.level == LEVEL_KNOWNCLASS: - if self.known_class.same_constant(other.known_class): - return - raise VirtualStatesCantMatch("classes don't match") - else: - assert other.level == LEVEL_CONSTANT - if (other.constbox.nonnull() and - self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))): - return - else: - raise VirtualStatesCantMatch("classes don't match") - + return self._generate_guards_unkown(other, box, runtime_box, + extra_guards, + state) else: + if not isinstance(other, NotVirtualStateInfo): + raise VirtualStatesCantMatch( + 'comparing a constant against something that is a virtual') assert self.level == LEVEL_CONSTANT if other.level == LEVEL_CONSTANT: if self.constbox.same_constant(other.constbox): @@ -485,19 +399,9 @@ raise VirtualStatesCantMatch("other not constant") assert 0, "unreachable" - def _generate_guards_intbounds(self, other, box, runtime_box, extra_guards, - optimizer): - if self.intbound is None: - return - if self.intbound.contains_bound(other.intbound): - return - if (runtime_box is not None and - self.intbound.contains(runtime_box.getint())): - # this may generate a few more guards than needed, but they are - # optimized away when emitting them - self.intbound.make_guards(box, extra_guards, optimizer) - return - raise VirtualStatesCantMatch("intbounds don't match") + def _generate_guards_unkown(self, other, box, runtime_box, extra_guards, + state): + return def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False): if self.level == LEVEL_CONSTANT: @@ -553,8 +457,145 @@ if self.lenbound: lb = ', ' + self.lenbound.bound.__repr__() - debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position + - ', ' + l + ', ' + self.intbound.__repr__() + lb + ')') + result = indent + mark + 'NotVirtualStateInfo(%d' % self.position + ', ' + l + extra = self._extra_repr() + if extra: + result += ', ' + extra + result += lb + ')' + debug_print(result) + +class NotVirtualStateInfoInt(NotVirtualStateInfo): + intbound = None + + def __init__(self, cpu, type, info): + NotVirtualStateInfo.__init__(self, cpu, type, info) + assert type == 'i' + if isinstance(info, IntBound): + if info.lower < MININT / 2: + info.lower = MININT + if info.upper > MAXINT / 2: + info.upper = MAXINT + self.intbound = info + + def _generate_guards_unkown(self, other, box, runtime_box, extra_guards, + optimizer): + other_intbound = None + if isinstance(other, NotVirtualStateInfoInt): + other_intbound = other.intbound + if self.intbound is None: + return + if self.intbound.contains_bound(other_intbound): + return + if (runtime_box is not None and + self.intbound.contains(runtime_box.getint())): + # this may generate a few more guards than needed, but they are + # optimized away when emitting them + self.intbound.make_guards(box, extra_guards, optimizer) + return + raise VirtualStatesCantMatch("intbounds don't match") + + def _extra_repr(self): + return self.intbound.__repr__() + + +class NotVirtualStateInfoPtr(NotVirtualStateInfo): + lenbound = None + known_class = None + + def __init__(self, cpu, type, info): + if info: + self.known_class = info.get_known_class(cpu) + if self.known_class: + self.level = LEVEL_KNOWNCLASS + elif info.is_nonnull(): + self.level = LEVEL_NONNULL + self.lenbound = info.getlenbound(None) + # might set it to LEVEL_CONSTANT + NotVirtualStateInfo.__init__(self, cpu, type, info) + + def _generate_guards(self, other, box, runtime_box, state): + if not isinstance(other, NotVirtualStateInfoPtr): + raise VirtualStatesCantMatch( + 'The VirtualStates does not match as a ' + + 'virtual appears where a pointer is needed ' + + 'and it is too late to force it.') + extra_guards = state.extra_guards + if self.lenbound: + if other.lenbound is None: + other_bound = IntLowerBound(0) + else: + other_bound = other.lenbound + if not self.lenbound.contains_bound(other_bound): + raise VirtualStatesCantMatch("length bound does not match") + if self.level == LEVEL_NONNULL: + return self._generate_guards_nonnull(other, box, runtime_box, + extra_guards, + state) + elif self.level == LEVEL_KNOWNCLASS: + return self._generate_guards_knownclass(other, box, runtime_box, + extra_guards, + state) + return NotVirtualStateInfo._generate_guards(self, other, box, + runtime_box, state) + + + # the following methods often peek into the runtime value that the + # box had when tracing. This value is only used as an educated guess. + # It is used here to choose between either emitting a guard and jumping + # to an existing compiled loop or retracing the loop. Both alternatives + # will always generate correct behaviour, but performance will differ. + + def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards, + state): + if not isinstance(other, NotVirtualStateInfoPtr): + raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!') + if other.level == LEVEL_UNKNOWN: + if runtime_box is not None and runtime_box.nonnull(): + op = ResOperation(rop.GUARD_NONNULL, [box]) + extra_guards.append(op) + return + else: + raise VirtualStatesCantMatch("other not known to be nonnull") + elif other.level == LEVEL_NONNULL: + pass + elif other.level == LEVEL_KNOWNCLASS: + pass # implies nonnull + else: + assert other.level == LEVEL_CONSTANT + assert other.constbox + if not other.constbox.nonnull(): + raise VirtualStatesCantMatch("constant is null") + + def _generate_guards_knownclass(self, other, box, runtime_box, extra_guards, + state): + cpu = state.cpu + if not isinstance(other, NotVirtualStateInfoPtr): + raise VirtualStatesCantMatch('trying to match ptr with non-ptr??!') + if other.level == LEVEL_UNKNOWN: + if (runtime_box and runtime_box.nonnull() and + self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))): + op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class]) + extra_guards.append(op) + else: + raise VirtualStatesCantMatch("other's class is unknown") + elif other.level == LEVEL_NONNULL: + if (runtime_box and self.known_class.same_constant( + cpu.ts.cls_of_box(runtime_box))): + op = ResOperation(rop.GUARD_CLASS, [box, self.known_class]) + extra_guards.append(op) + else: + raise VirtualStatesCantMatch("other's class is unknown") + elif other.level == LEVEL_KNOWNCLASS: + if self.known_class.same_constant(other.known_class): + return + raise VirtualStatesCantMatch("classes don't match") + else: + assert other.level == LEVEL_CONSTANT + if (other.constbox.nonnull() and + self.known_class.same_constant(cpu.ts.cls_of_box(other.constbox))): + pass + else: + raise VirtualStatesCantMatch("classes don't match") class VirtualState(object): @@ -678,8 +719,8 @@ return VirtualState(state) def visit_not_virtual(self, box): - return NotVirtualStateInfo(self.optimizer.cpu, box.type, - self.optimizer.getinfo(box)) + return not_virtual(self.optimizer.cpu, box.type, + self.optimizer.getinfo(box)) def visit_virtual(self, descr, fielddescrs): known_class = ConstInt(descr.get_vtable()) diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -291,6 +291,7 @@ return _search(value, other, start, end, SEARCH_COUNT) # -------------- substring searching helper ---------------- +# XXX a lot of code duplication with lltypesystem.rstr :-( SEARCH_COUNT = 0 SEARCH_FIND = 1 @@ -309,6 +310,8 @@ if end > len(value): end = len(value) if start > end: + if mode == SEARCH_COUNT: + return 0 return -1 count = 0 @@ -326,6 +329,8 @@ w = n - m if w < 0: + if mode == SEARCH_COUNT: + return 0 return -1 mlast = m - 1 @@ -570,18 +575,20 @@ class ByteListBuilder(object): def __init__(self, init_size=INIT_SIZE): + assert init_size >= 0 self.l = newlist_hint(init_size) @specialize.argtype(1) def append(self, s): + l = self.l for c in s: - self.l.append(c) + l.append(c) @specialize.argtype(1) def append_slice(self, s, start, end): - assert 0 <= start <= end <= len(s) - for c in s[start:end]: - self.l.append(c) + l = self.l + for i in xrange(start, end): + l.append(s[i]) def append_multiple_char(self, c, times): assert isinstance(c, str) @@ -589,8 +596,9 @@ def append_charpsize(self, s, size): assert size >= 0 + l = self.l for i in xrange(size): - self.l.append(s[i]) + l.append(s[i]) def build(self): return self.l diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py --- a/rpython/rlib/test/test_rstring.py +++ b/rpython/rlib/test/test_rstring.py @@ -231,6 +231,10 @@ check_search(count, 'one two three', 'e', 0, 1, res=0) check_search(count, 'one two three', '', 0, 13, res=14) + check_search(count, '', 'ab', 0, 0, res=0) + check_search(count, 'a', 'ab', 0, 1, res=0) + check_search(count, 'ac', 'ab', 0, 2, res=0) + class TestTranslates(BaseRtypingTest): def test_split_rsplit(self): diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -972,6 +972,13 @@ s.count(s, -10) py.test.raises(AnnotatorError, self.interpret, f, ()) + def test_count_in_empty_string(self): + const = self.const + def fn(): + return const('').count(const('ab')) + res = self.interpret(fn, []) + assert res == 0 + def test_getitem_exc(self): const = self.const def f(x): From pypy.commits at gmail.com Wed Jun 22 10:44:13 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 22 Jun 2016 07:44:13 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: (ppc) added several new ppc op codes, Message-ID: <576aa43d.67c0c20a.bcb50.5a99@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85328:7799b9b5dc53 Date: 2016-06-22 16:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7799b9b5dc53/ Log: (ppc) added several new ppc op codes, fixed endian issues for vec load/store (int only). added more tests for integer add diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -614,11 +614,18 @@ # INTEGER # ------- - # load & store + # load lvx = XV(31, XO1=103) + lvewx = XV(31, XO1=71) + lvehx = XV(31, XO1=39) + lvebx = XV(31, XO1=7) + # store stvx = XV(31, XO1=231) + stvewx = XV(31, XO1=199) + stvehx = XV(31, XO1=167) + stvebx = XV(31, XO1=135) - # arith & logic + # arith vaddudm = VX(4, XO8=192) vadduwm = VX(4, XO8=128) vadduhm = VX(4, XO8=64) @@ -629,6 +636,14 @@ vsubuhm = VX(4, XO8=1088) vsububm = VX(4, XO8=1024) + # logic + vand = VX(4, XO8=1028) + vor = VX(4, XO8=1156) + veqv = VX(4, XO8=1668) + + # vector move register is alias to vector or + vmr = vor + # shift, perm and select diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py --- a/rpython/jit/backend/ppc/locations.py +++ b/rpython/jit/backend/ppc/locations.py @@ -30,6 +30,9 @@ def is_fp_reg(self): return False + def is_vector_reg(self): + return False + def is_imm_float(self): return False @@ -77,7 +80,7 @@ class VectorRegisterLocation(AssemblerLocation): _immutable_ = True - width = WORD + width = WORD * 2 type = VECTOR def __init__(self, value): @@ -92,6 +95,9 @@ def as_key(self): return self.value + 132 + def is_vector_reg(self): + return True + class ImmLocation(AssemblerLocation): _immutable_ = True diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py --- a/rpython/jit/backend/ppc/ppc_assembler.py +++ b/rpython/jit/backend/ppc/ppc_assembler.py @@ -808,6 +808,8 @@ # name = "Loop # %s: %s" % (looptoken.number, loopname) # self.cpu.profile_agent.native_code_written(name, # rawstart, full_size) + #print(hex(rawstart)) + #import pdb; pdb.set_trace() return AsmInfo(ops_offset, rawstart + looppos, size_excluding_failure_stuff - looppos) @@ -1044,6 +1046,10 @@ self.mc.lfd(reg, r.SPP.value, offset) return assert 0, "not supported location" + elif prev_loc.is_vector_reg(): + assert loc.is_vector_reg() + self.mc.vmr(loc.value, prev_loc.value, prev_loc.value) + return elif prev_loc.is_reg(): reg = prev_loc.value # move to another register diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -11,6 +11,7 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem import lltype from rpython.jit.backend.ppc.locations import imm +from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN def not_implemented(msg): msg = '[ppc/vector_ext] %s\n' % msg @@ -46,6 +47,34 @@ elif itemsize == 8: self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value) + def dispatch_vector_load(self, size, Vt, index, addr): + self.mc.lvx(Vt, index, addr) + return + if size == 8: + self.mc.lvx(Vt, index, addr) + elif size == 4: + self.mc.lvewx(Vt, index, addr) + elif size == 2: + self.mc.lvehx(Vt, index, addr) + elif size == 1: + self.mc.lvehx(Vt, index, addr) + else: + notimplemented("[ppc/assembler] dispatch vector load of size %d" % size) + + def dispatch_vector_store(self, size, Vt, index, addr): + self.mc.stvx(Vt, index, addr) + return + if size == 8: + self.mc.stvx(Vt, index, addr) + elif size == 4: + self.mc.stvewx(Vt, index, addr) + elif size == 2: + self.mc.stvehx(Vt, index, addr) + elif size == 1: + self.mc.stvehx(Vt, index, addr) + else: + notimplemented("[ppc/assembler] dispatch vector load of size %d" % size) + def emit_vec_raw_load_i(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, \ Vhiloc, Vloloc, Vploc, tloc = arglocs @@ -56,10 +85,17 @@ self.mc.lvx(Vhi, indexloc.value, baseloc.value) Vp = Vploc.value t = tloc.value - self.mc.lvsl(Vp, indexloc.value, baseloc.value) + if IS_BIG_ENDIAN: + self.mc.lvsl(Vp, indexloc.value, baseloc.value) + else: + self.mc.lvsr(Vp, indexloc.value, baseloc.value) self.mc.addi(t, baseloc.value, 16) self.mc.lvx(Vlo, indexloc.value, t) - self.mc.vperm(resloc.value, Vhi, Vlo, Vp) + if IS_BIG_ENDIAN: + self.mc.vperm(resloc.value, Vhi, Vlo, Vp) + else: + self.mc.vperm(resloc.value, Vlo, Vhi, Vp) + #self.mc.trap() def _emit_vec_setitem(self, op, arglocs, regalloc): # prepares item scale (raw_store does not) @@ -101,12 +137,19 @@ # probably a lot of room for improvement (not locally, # but in general for the algorithm) self.mc.lvx(Vhi, indexloc.value, baseloc.value) - self.mc.lvsr(Vp, indexloc.value, baseloc.value) + #self.mc.lvsr(Vp, indexloc.value, baseloc.value) + if IS_BIG_ENDIAN: + self.mc.lvsr(Vp, indexloc.value, baseloc.value) + else: + self.mc.lvsl(Vp, indexloc.value, baseloc.value) self.mc.addi(t, baseloc.value, 16) self.mc.lvx(Vlo, indexloc.value, t) self.mc.vspltisb(V1s, -1) self.mc.vspltisb(V0s, 0) - self.mc.vperm(Vmask, V0s, V1s, Vp) + if IS_BIG_ENDIAN: + self.mc.vperm(Vmask, V0s, V1s, Vp) + else: + self.mc.vperm(Vmask, V1s, V0s, Vp) self.mc.vperm(Vs, Vs, Vs, Vp) self.mc.vsel(Vlo, Vs, Vlo, Vmask) self.mc.vsel(Vhi, Vhi, Vs, Vmask) @@ -179,27 +222,24 @@ self.mc.xvdivdp(resloc.value, loc0.value, loc1.value) def emit_vec_int_mul(self, op, arglocs, resloc): - loc0, loc1, itemsize_loc = arglocs - itemsize = itemsize_loc.value - if itemsize == 1: - self.mc.PMULLW(loc0, loc1) - elif itemsize == 2: - self.mc.PMULLW(loc0, loc1) - elif itemsize == 4: - self.mc.PMULLD(loc0, loc1) - else: - # NOTE see http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025 - # There is no 64x64 bit packed mul. For 8 bit either. It is questionable if it gives any benefit? - not_implemented("int8/64 mul") + pass # TODO - def emit_vec_int_and(self, op, arglocs, resloc): - self.mc.PAND(resloc, arglocs[0]) + def emit_vec_int_and(self, op, arglocs, regalloc): + resloc, loc0, loc1 = arglocs + self.mc.vand(resloc.value, loc0.value, loc1.value) - def emit_vec_int_or(self, op, arglocs, resloc): - self.mc.POR(resloc, arglocs[0]) + def emit_vec_int_or(self, op, arglocs, regalloc): + resloc, loc0, loc1 = arglocs + self.mc.vor(resloc.value, loc0.value, loc1.value) - def emit_vec_int_xor(self, op, arglocs, resloc): - self.mc.PXOR(resloc, arglocs[0]) + def emit_vec_int_xor(self, op, arglocs, regalloc): + resloc, loc0, loc1 = arglocs + self.mc.veqv(resloc.value, loc0.value, loc1.value) + + def emit_vec_int_signext(self, op, arglocs, regalloc): + resloc, loc0 = arglocs + # TODO + self.regalloc_mov(loc0, resloc) #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc): # self.implement_guard(guard_token) @@ -367,34 +407,6 @@ # # ----------- pxor # # 00 11 00 00 - #def genop_vec_int_signext(self, op, arglocs, resloc): - # srcloc, sizeloc, tosizeloc = arglocs - # size = sizeloc.value - # tosize = tosizeloc.value - # if size == tosize: - # return # already the right size - # if size == 4 and tosize == 8: - # scratch = X86_64_SCRATCH_REG.value - # self.mc.PEXTRD_rxi(scratch, srcloc.value, 1) - # self.mc.PINSRQ_xri(resloc.value, scratch, 1) - # self.mc.PEXTRD_rxi(scratch, srcloc.value, 0) - # self.mc.PINSRQ_xri(resloc.value, scratch, 0) - # elif size == 8 and tosize == 4: - # # is there a better sequence to move them? - # scratch = X86_64_SCRATCH_REG.value - # self.mc.PEXTRQ_rxi(scratch, srcloc.value, 0) - # self.mc.PINSRD_xri(resloc.value, scratch, 0) - # self.mc.PEXTRQ_rxi(scratch, srcloc.value, 1) - # self.mc.PINSRD_xri(resloc.value, scratch, 1) - # else: - # # note that all other conversions are not implemented - # # on purpose. it needs many x86 op codes to implement - # # the missing combinations. even if they are implemented - # # the speedup might only be modest... - # # the optimization does not emit such code! - # msg = "vec int signext (%d->%d)" % (size, tosize) - # not_implemented(msg) - #def genop_vec_expand_f(self, op, arglocs, resloc): # srcloc, sizeloc = arglocs # size = sizeloc.value @@ -666,6 +678,14 @@ prepare_vec_raw_store = _prepare_vec_store del _prepare_vec_store + def prepare_vec_int_signext(self, op): + assert isinstance(op, VectorOp) + a0 = op.getarg(0) + loc0 = self.ensure_vector_reg(a0) + resloc = self.force_allocate_vector_reg(op) + return [resloc, loc0] + + #def prepare_vec_arith_unary(self, op): # lhs = op.getarg(0) @@ -758,16 +778,6 @@ # resloc = self.xrm.force_allocate_reg(op, args) # self.perform(op, [srcloc, imm(op.bytesize)], resloc) - #def prepare_vec_int_signext(self, op): - # assert isinstance(op, VectorOp) - # args = op.getarglist() - # resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # arg = op.getarg(0) - # assert isinstance(arg, VectorOp) - # size = arg.bytesize - # assert size > 0 - # self.perform(op, [resloc, imm(size), imm(op.bytesize)], resloc) - #def prepare_vec_int_is_true(self, op): # args = op.getarglist() # arg = op.getarg(0) diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -1,6 +1,7 @@ import py import pytest import math +import functools from hypothesis import given, note, strategies as st from rpython.jit.metainterp.warmspot import ll_meta_interp, get_stats from rpython.jit.metainterp.test.support import LLJitMixin @@ -11,7 +12,7 @@ from rpython.rlib.objectmodel import compute_hash from rpython.rlib import rfloat from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_uint, intmask, r_int from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, free_raw_storage, raw_storage_getitem) from rpython.rlib.objectmodel import (specialize, is_annotation_constant, @@ -59,8 +60,6 @@ request.cls.a return rs -integers_64bit = st.integers(min_value=-2**63, max_value=2**63-1) -floats = st.floats() def rdiv(v1,v2): # TODO unused, interpeting this on top of llgraph does not work correctly @@ -104,11 +103,11 @@ raw_storage_setitem(vc, i, rffi.cast(type,c)) i += size - la = data.draw(st.lists(floats, min_size=10, max_size=150)) + la = data.draw(st.lists(st.floats(), min_size=10, max_size=150)) #la = [0.0,0.0,0.0,0.0,0.0,0.0,0.0] #lb = [0.0,0.0,0.0,0.0,1.7976931348623157e+308,0.0,0.0] l = len(la) - lb = data.draw(st.lists(floats, min_size=l, max_size=l)) + lb = data.draw(st.lists(st.floats(), min_size=l, max_size=l)) rawstorage = RawStorage() va = rawstorage.new(la, type) @@ -126,18 +125,7 @@ rawstorage.clear() - #@given(st.data()) - @pytest.mark.parametrize('func,type', [ - (lambda a,b: intmask(a+b), rffi.SIGNED), - (lambda a,b: intmask(a+b), rffi.UNSIGNED), - (lambda a,b: intmask(a+b), rffi.INT), - (lambda a,b: intmask(a+b), rffi.UINT), - (lambda a,b: intmask(a+b), rffi.SHORT), - (lambda a,b: intmask(a+b), rffi.USHORT), - (lambda a,b: intmask(a+b), rffi.CHAR), - (lambda a,b: intmask(a+b), rffi.UCHAR), - ]) - def test_vector_simple_int(self, func, type): + def _vector_simple_int(self, func, type, data): func = always_inline(func) size = rffi.sizeof(type) @@ -152,11 +140,13 @@ raw_storage_setitem(vc, i, rffi.cast(type,c)) i += size - #la = data.draw(st.lists(integers_64bit, min_size=10, max_size=150)) - la = [1] * 10 + bits = size*8 + integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1) + la = data.draw(st.lists(integers, min_size=10, max_size=150)) + #la = [1,2,3,4,5,6,7,8,9,10,11,12,13] l = len(la) - #lb = data.draw(st.lists(integers_64bit, min_size=l, max_size=l)) - lb = [0] * 10 + #lb = [1,2,3,4,5,6,7,8,9,10,11,12,13] + lb = data.draw(st.lists(integers, min_size=l, max_size=l)) rawstorage = RawStorage() va = rawstorage.new(la, type) @@ -166,10 +156,31 @@ for i in range(l): c = raw_storage_getitem(type,vc,i*size) - assert func(la[i], lb[i]) == c + assert rffi.cast(type, func(la[i], lb[i])) == c rawstorage.clear() + def vec_int_arith(test_func, arith_func, type): + return pytest.mark.parametrize('func,type', [ + (arith_func, type) + ])(given(data=st.data())(test_func)) + + vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int) + + test_vector_signed_add = \ + vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED) + test_vector_int_add = \ + vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT) + test_vector_short_add = \ + vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT) + + test_vector_signed_sub = \ + vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SIGNED) + test_vector_int_sub = \ + vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT) + test_vector_short_sub = \ + vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT) + @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,131,142,143]) def test_vectorize_array_get_set(self,i): myjitdriver = JitDriver(greens = [], From pypy.commits at gmail.com Wed Jun 22 11:29:58 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 22 Jun 2016 08:29:58 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: added test to check &, | and ^ vector operations, +impl. tests pass Message-ID: <576aaef6.69fac20a.5bcff.ffffd18f@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85329:427a2b9a7a9b Date: 2016-06-22 17:28 +0200 http://bitbucket.org/pypy/pypy/changeset/427a2b9a7a9b/ Log: added test to check &,| and ^ vector operations, +impl. tests pass diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -47,34 +47,6 @@ elif itemsize == 8: self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value) - def dispatch_vector_load(self, size, Vt, index, addr): - self.mc.lvx(Vt, index, addr) - return - if size == 8: - self.mc.lvx(Vt, index, addr) - elif size == 4: - self.mc.lvewx(Vt, index, addr) - elif size == 2: - self.mc.lvehx(Vt, index, addr) - elif size == 1: - self.mc.lvehx(Vt, index, addr) - else: - notimplemented("[ppc/assembler] dispatch vector load of size %d" % size) - - def dispatch_vector_store(self, size, Vt, index, addr): - self.mc.stvx(Vt, index, addr) - return - if size == 8: - self.mc.stvx(Vt, index, addr) - elif size == 4: - self.mc.stvewx(Vt, index, addr) - elif size == 2: - self.mc.stvehx(Vt, index, addr) - elif size == 1: - self.mc.stvehx(Vt, index, addr) - else: - notimplemented("[ppc/assembler] dispatch vector load of size %d" % size) - def emit_vec_raw_load_i(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, \ Vhiloc, Vloloc, Vploc, tloc = arglocs @@ -225,15 +197,15 @@ pass # TODO def emit_vec_int_and(self, op, arglocs, regalloc): - resloc, loc0, loc1 = arglocs + resloc, loc0, loc1, sizeloc = arglocs self.mc.vand(resloc.value, loc0.value, loc1.value) def emit_vec_int_or(self, op, arglocs, regalloc): - resloc, loc0, loc1 = arglocs + resloc, loc0, loc1, sizeloc = arglocs self.mc.vor(resloc.value, loc0.value, loc1.value) def emit_vec_int_xor(self, op, arglocs, regalloc): - resloc, loc0, loc1 = arglocs + resloc, loc0, loc1, sizeloc = arglocs self.mc.veqv(resloc.value, loc0.value, loc1.value) def emit_vec_int_signext(self, op, arglocs, regalloc): diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -181,6 +181,27 @@ test_vector_short_sub = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT) + test_vector_signed_and = \ + vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SIGNED) + test_vector_int_and = \ + vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.INT) + test_vector_short_and = \ + vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SHORT) + + test_vector_or_signed = \ + vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SIGNED) + test_vector_or_int = \ + vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.INT) + test_vector_or_short = \ + vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SHORT) + + test_vector_xor_signed = \ + vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SIGNED) + test_vector_xor_int = \ + vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.INT) + test_vector_xor_short = \ + vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT) + @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,131,142,143]) def test_vectorize_array_get_set(self,i): myjitdriver = JitDriver(greens = [], From pypy.commits at gmail.com Wed Jun 22 11:43:45 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 22 Jun 2016 08:43:45 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: (ppc) added hypothesis test for vec_float_abs Message-ID: <576ab231.2523c20a.e0232.ffffe7b7@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85330:7365988a68a4 Date: 2016-06-22 17:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7365988a68a4/ Log: (ppc) added hypothesis test for vec_float_abs diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -84,6 +84,49 @@ type_system=self.type_system, vec=vec, vec_all=vec_all) + def _vector_float_unary(self, func, type, data): + func = always_inline(func) + + size = rffi.sizeof(type) + myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) + def f(bytecount, va, vc): + i = 0 + while i < bytecount: + myjitdriver.jit_merge_point() + a = raw_storage_getitem(type,va,i) + c = func(a) + raw_storage_setitem(vc, i, rffi.cast(type,c)) + i += size + + la = data.draw(st.lists(st.floats(), min_size=10, max_size=150)) + l = len(la) + + rawstorage = RawStorage() + va = rawstorage.new(la, type) + vc = rawstorage.new(None, type, size=l) + self.meta_interp(f, [l*size, va, vc]) + + for i in range(l): + c = raw_storage_getitem(type,vc,i*size) + r = func(la[i]) + assert isclose(r, c) or (math.isnan(r) and math.isnan(c)) or \ + (math.isinf(r) and math.isinf(c) and \ + (r < 0.0 and c < 0.0) or \ + (r > 0.0 and c > 0.0)) + + rawstorage.clear() + + def vec_int_unary(test_func, unary_func, type): + return pytest.mark.parametrize('func,type', [ + (unary_func, type) + ])(given(data=st.data())(test_func)) + + vec_float_unary = functools.partial(vec_int_unary, _vector_float_unary) + + test_vec_abs_float = \ + vec_float_unary(lambda v: abs(v), rffi.DOUBLE) + + @given(data=st.data()) @pytest.mark.parametrize('func', [lambda a,b: a+b, lambda a,b: a*b, lambda a,b: a-b]) From pypy.commits at gmail.com Wed Jun 22 12:24:25 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 09:24:25 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Throw error if unpacking is used in comprehension Message-ID: <576abbb9.d12f1c0a.e1528.ffff8d47@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85331:3c7f568374d8 Date: 2016-06-22 18:23 +0200 http://bitbucket.org/pypy/pypy/changeset/3c7f568374d8/ Log: Throw error if unpacking is used in comprehension diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1315,18 +1315,24 @@ def handle_genexp(self, genexp_node): elt = self.handle_expr(genexp_node.get_child(0)) + if elt.type == syms.star_expr: + self.error("iterable unpacking cannot be used in comprehension", elt) comps = self.comprehension_helper(genexp_node.get_child(1)) return ast.GeneratorExp(elt, comps, genexp_node.get_lineno(), genexp_node.get_column()) def handle_listcomp(self, listcomp_node): elt = self.handle_expr(listcomp_node.get_child(0)) + if elt.type == syms.star_expr: + self.error("iterable unpacking cannot be used in comprehension", elt) comps = self.comprehension_helper(listcomp_node.get_child(1)) return ast.ListComp(elt, comps, listcomp_node.get_lineno(), listcomp_node.get_column()) def handle_setcomp(self, set_maker): elt = self.handle_expr(set_maker.get_child(0)) + if elt.type == syms.star_expr: + self.error("iterable unpacking cannot be used in comprehension", elt) comps = self.comprehension_helper(set_maker.get_child(1)) return ast.SetComp(elt, comps, set_maker.get_lineno(), set_maker.get_column()) From pypy.commits at gmail.com Wed Jun 22 13:10:06 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 10:10:06 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Handle dictdisplay in ast Message-ID: <576ac66e.c7a81c0a.f7e9c.380f@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85332:708d898b26f4 Date: 2016-06-22 19:09 +0200 http://bitbucket.org/pypy/pypy/changeset/708d898b26f4/ Log: Handle dictdisplay in ast diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1338,8 +1338,7 @@ set_maker.get_column()) def handle_dictcomp(self, dict_maker): - i = 0 - dictelement = self.handle_dictelement(dict_maker, i) + dictelement = self.handle_dictelement(dict_maker, 0) i = dictelement[0] key = dictelement[1] value = dictelement[2] @@ -1348,8 +1347,15 @@ dict_maker.get_column()) def handle_dictdisplay(self, node): - size = (node.num_children()+1) / 3 - + keys = [] + values = [] + i = 0 + while i < node.num_children(): + dictelement = self.handle_dictelement(node, i) + i = dictelement[0] + keys.append(dictelement[1]) + values.append(dictelement[2]) + i += 1 return ast.Dict(keys, values, node.get_lineno(), node.get_column()) def handle_exprlist(self, exprlist, context): From pypy.commits at gmail.com Wed Jun 22 13:22:55 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 10:22:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Handle setdisplay in ast Message-ID: <576ac96f.9a4a1c0a.35bba.ffff9fc6@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85333:002348cdc66f Date: 2016-06-22 19:22 +0200 http://bitbucket.org/pypy/pypy/changeset/002348cdc66f/ Log: Handle setdisplay in ast diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1357,6 +1357,15 @@ values.append(dictelement[2]) i += 1 return ast.Dict(keys, values, node.get_lineno(), node.get_column()) + + def handle_setdisplay(self, node): + elts = [] + i = 0 + while i < node.num_children(): + expr = self.handle_expr(node.get_child(i)) + elts.append(expr) + i += 2 + return ast.Set(elts, node.get_lineno(), node.get_column()) def handle_exprlist(self, exprlist, context): exprs = [] From pypy.commits at gmail.com Wed Jun 22 14:26:49 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 11:26:49 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix keyword argument unpacking in handle_call Message-ID: <576ad869.c5ddc20a.d9c04.3811@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85334:d3ccf823b126 Date: 2016-06-22 20:25 +0200 http://bitbucket.org/pypy/pypy/changeset/d3ccf823b126/ Log: Fix keyword argument unpacking in handle_call diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1008,7 +1008,6 @@ def handle_call(self, args_node, callable_expr): arg_count = 0 # position args + iterable args unpackings keyword_count = 0 # keyword args + keyword args unpackings - doublestars_count = 0 # just keyword argument unpackings generator_count = 0 for i in range(args_node.num_children()): argument = args_node.get_child(i) @@ -1032,6 +1031,7 @@ args = [] keywords = [] used_keywords = {} + doublestars_count = 0 # just keyword argument unpackings child_count = args_node.num_children() i = 0 while i < child_count: @@ -1062,8 +1062,9 @@ expr_node.get_column())) elif expr_node.type == tokens.DOUBLESTAR: # a keyword argument unpacking + i += 1 expr = self.handle_expr(argument.get_child(1)) - args.append(ast.keyword(None, expr)) + keywords.append(ast.keyword(None, expr)) doublestars_count += 1 elif argument.get_child(1).type == syms.comp_for: # the lone generator expression @@ -1223,7 +1224,7 @@ elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for: # a set comprehension return self.handle_setcomp(maker) - elif (n_maker_children > 3 - is_dict and + elif (n_maker_children > (3-is_dict) and maker.get_child(3-is_dict).type == syms.comp_for): # a dictionary comprehension if is_dict: From pypy.commits at gmail.com Wed Jun 22 14:31:18 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 22 Jun 2016 11:31:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: watchpoints, in-progress Message-ID: <576ad976.c7a81c0a.400fd.04be@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85335:fb7901990151 Date: 2016-06-22 17:30 +0200 http://bitbucket.org/pypy/pypy/changeset/fb7901990151/ Log: watchpoints, in-progress diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -16,12 +16,12 @@ CMD_BREAKPOINTS = 4 CMD_MOREINFO = 5 CMD_ATTACHID = 6 -CMD_WATCH = 7 -CMD_EXPECTED = 8 +CMD_CHECKWATCH = 7 +CMD_WATCHVALUES = 8 ANSWER_TEXT = 20 ANSWER_MOREINFO = 21 ANSWER_NEXTNID = 22 -ANSWER_COMPILED = 23 +ANSWER_WATCH = 23 def stop_point(): @@ -46,6 +46,9 @@ def send_nextnid(unique_id): send_answer(ANSWER_NEXTNID, unique_id) +def send_watch(text, ok_flag): + send_answer(ANSWER_WATCH, ok_flag, extra=text) + def current_time(): """For RPython debug commands: returns the current time.""" return llop.revdb_get_value(lltype.SignedLongLong, 'c') @@ -103,6 +106,12 @@ ll_callback = llhelper(_CALLBACK_GCREF_FNPTR, callback) llop.revdb_track_object(lltype.Void, unique_id, ll_callback) +def save_state(): + return llop.revdb_save_state(lltype.Bool) + +def restore_state(): + llop.revdb_restore_state(lltype.Void) + # ____________________________________________________________ @@ -157,7 +166,7 @@ llannotation.lltype_to_annotation(llmemory.GCREF)] def arguments_WATCHING(self): - return [] + raise Exception("XXX remove me") def specialize_call(self, hop): hop.exception_cannot_occur() diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -571,7 +571,8 @@ 'revdb_breakpoint': LLOp(), 'revdb_get_value': LLOp(sideeffects=False), 'revdb_get_unique_id': LLOp(sideeffects=False), - ## 'revdb_track_object': LLOp(), + 'revdb_save_state': LLOp(), + 'revdb_restore_state': LLOp(), } # ***** Run test_lloperation after changes. ***** diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -27,7 +27,6 @@ lltype.Void)) ALLOCFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.LONGLONG, llmemory.GCREF], lltype.Void)) - WATCHFUNCPTR = lltype.Ptr(lltype.FuncType([], lltype.Signed)) bk = db.translator.annotator.bookkeeper cmds = getattr(db.translator, 'revdb_commands', {}) @@ -37,8 +36,7 @@ S = lltype.Struct('RPY_REVDB_COMMANDS', ('names', lltype.FixedSizeArray(rffi.INT, len(numcmds) + 1)), ('funcs', lltype.FixedSizeArray(FUNCPTR, len(numcmds))), - ('alloc', ALLOCFUNCPTR), - ('watch', WATCHFUNCPTR)) + ('alloc', ALLOCFUNCPTR)) s = lltype.malloc(S, flavor='raw', immortal=True, zero=True) i = 0 @@ -51,8 +49,6 @@ i += 1 elif name == "ALLOCATING": s.alloc = fnptr - elif name == "WATCHING": - s.watch = fnptr else: raise AssertionError("bad tag in register_debug_command(): %r" % (name,)) diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -109,14 +109,34 @@ lst = [str(n) for n in sorted(self.pgroup.paused)] print ', '.join(lst) + def _bp_kind(self, name): + if name[0] == 'B': + return 'breakpoint' + elif name[0] == 'W': + return 'watchpoint' + else: + return '?????point' + + def _bp_new(self, break_at, watchvalue=None): + b = self.pgroup.edit_breakpoints() + new = 1 + while new in b.num2name: + new += 1 + b.num2name[new] = break_at + if watchvalue is not None: + b.watchvalues[new] = watchvalue + print "%s %d added" % (self._bp_kind(break_at).capitalize(), new) + return new + def cmd_info_breakpoints(self): - """List current breakpoints""" + """List current breakpoints and watchpoints""" lst = self.pgroup.all_breakpoints.num2name.items() if lst: for num, name in sorted(lst): - print '%8d: %s' % (num, name) + print '\t%s %d: %s' % (self._bp_kind(name), num, name[1:]) else: print 'no breakpoints.' + cmd_info_watchpoints = cmd_info_breakpoints def move_forward(self, steps): self.remove_tainting() @@ -137,7 +157,10 @@ def hit_breakpoint(self, b, backward=False): if b.num != -1: - self.print_extra_pending_info = 'Hit breakpoint %d' % (b.num,) + name = self.pgroup.all_breakpoints.num2name.get(b.num, '??') + kind = self._bp_kind(name) + self.print_extra_pending_info = 'Hit %s %d: %s' % (kind, b.num, + name[1:]) elif backward: b.time -= 1 if self.pgroup.get_current_time() != b.time: @@ -240,20 +263,30 @@ if not argument: print "Break where?" return - b = self.pgroup.edit_breakpoints() - new = 1 - while new in b.num2name: - new += 1 - b.num2name[new] = argument - print "Breakpoint %d added" % (new,) + self._bp_new('B' + argument) command_b = command_break def command_delete(self, argument): - """Delete a breakpoint""" + """Delete a breakpoint/watchpoint""" arg = int(argument) b = self.pgroup.edit_breakpoints() if arg not in b.num2name: - print "No breakpoint number %d" % (arg,) + print "No breakpoint/watchpoint number %d" % (arg,) else: - del b.num2name[arg] - print "Breakpoint %d deleted" % (arg,) + name = b.num2name.pop(arg) + b.watchvalues.pop(arg, '') + kind = self._bp_kind(name) + print "%s %d deleted: %s" % (kind.capitalize(), arg, name[1:]) + + def command_watch(self, argument): + """Add a watchpoint (use $NUM in the expression to watch)""" + if not argument: + print "Watch what?" + return + ok_flag, text = self.pgroup.check_watchpoint_expr(argument) + if not ok_flag: + print text + print 'Watchpoint not added' + else: + print 'Current value:', text + self._bp_new('W' + argument, watchvalue=text) diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -17,6 +17,8 @@ # extra="\0-separated names") CMD_MOREINFO = 5 # Message(CMD_MOREINFO) CMD_ATTACHID = 6 # Message(CMD_ATTACHID, small-num, unique-id) +CMD_CHECKWATCH = 7 # Message(CMD_CHECKWATCH, extra=expression) +CMD_WATCHVALUES = 8 # Message(CMD_WATCHVALUES, extra=texts) # the first message sent by the first child: @@ -56,6 +58,10 @@ # Message(ANSWER_NEXTNID, unique-id) ANSWER_NEXTNID = 22 +# sent after CMD_CHECKWATCH: +# Message(ANSWER_WATCH, ok_flag, extra=result_of_expr) +ANSWER_WATCH = 23 + # ____________________________________________________________ diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -18,18 +18,22 @@ class AllBreakpoints(object): def __init__(self): - self.num2name = {} # {small number: function name} - self.stack_depth = 0 # breaks if the depth becomes lower than this + self.num2name = {} # {small number: break/watchpoint} + self.watchvalues = {} # {small number: resulting text} + self.stack_depth = 0 # breaks if the depth becomes lower than this def __repr__(self): return 'AllBreakpoints(%r, %d)' % (self.num2name, self.stack_depth) - def __eq__(self, other): - return (self.num2name == other.num2name and - self.stack_depth == other.stack_depth) - - def __ne__(self, other): - return not (self == other) + def compare(self, other): + if (self.num2name == other.num2name and + self.stack_depth == other.stack_depth): + if self.watchvalues == other.watchvalues: + return 2 # completely equal + else: + return 1 # equal, but watchvalues out-of-date + else: + return 0 # different def is_empty(self): return len(self.num2name) == 0 and self.stack_depth == 0 @@ -71,7 +75,7 @@ return ''.join(pieces) def send(self, msg): - #print 'SENT:', self.pid, msg + print 'SENT:', self.pid, msg binary = struct.pack("iIqqq", msg.cmd, len(msg.extra), msg.arg1, msg.arg2, msg.arg3) self.control_socket.sendall(binary + msg.extra) @@ -81,7 +85,7 @@ cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary) extra = self._recv_all(size) msg = Message(cmd, arg1, arg2, arg3, extra) - #print 'RECV:', self.pid, msg + print 'RECV:', self.pid, msg return msg def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""): @@ -129,7 +133,7 @@ except socket.error: pass - def forward(self, steps, breakpoint_mode='b'): + def forward(self, steps, breakpoint_mode, all_breakpoints): """Move this subprocess forward in time. Returns the Breakpoint or None. """ @@ -145,6 +149,8 @@ break if bkpt is None: bkpt = Breakpoint(msg.arg1, msg.arg3) + all_breakpoints.watchvalues = dict.fromkeys( + all_breakpoints.watchvalues) # set all values to None assert msg.cmd == ANSWER_READY self.update_times(msg) return bkpt @@ -249,7 +255,8 @@ 'r' = record the occurrence of a breakpoint but continue """ assert steps >= 0 - self.update_breakpoints() + if breakpoint_mode != 'i': + self.update_breakpoints() latest_bkpt = None while True: cur_time = self.get_current_time() @@ -261,7 +268,8 @@ break assert rel_next_clone >= 0 if rel_next_clone > 0: - bkpt = self.active.forward(rel_next_clone, breakpoint_mode) + bkpt = self.active.forward(rel_next_clone, breakpoint_mode, + self.all_breakpoints) if breakpoint_mode == 'r': latest_bkpt = bkpt or latest_bkpt elif bkpt: @@ -271,7 +279,8 @@ self.paused[self.active.current_time].close() clone = self.active.clone() self.paused[clone.current_time] = clone - bkpt = self.active.forward(steps, breakpoint_mode) + bkpt = self.active.forward(steps, breakpoint_mode, + self.all_breakpoints) if breakpoint_mode == 'r': bkpt = bkpt or latest_bkpt if bkpt: @@ -313,19 +322,50 @@ search_start_time -= time_range_to_search * 3 def update_breakpoints(self): - if self.active.breakpoints_cache == self.all_breakpoints: - return + cmp = self.all_breakpoints.compare(self.active.breakpoints_cache) + print 'compare:', cmp, self.all_breakpoints.watchvalues + if cmp == 2: + return # up-to-date + + # update the breakpoints/watchpoints + self.active.breakpoints_cache = None num2name = self.all_breakpoints.num2name - flat = [] - if num2name: - flat = [num2name.get(n, '') for n in range(max(num2name) + 1)] - arg1 = self.all_breakpoints.stack_depth - extra = '\x00'.join(flat) - # - self.active.breakpoints_cache = None - self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra)) + N = (max(num2name) + 1) if num2name else 0 + if cmp == 0: + flat = [num2name.get(n, '') for n in range(N)] + arg1 = self.all_breakpoints.stack_depth + extra = '\x00'.join(flat) + self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra)) + self.active.expect_ready() + else: + assert cmp == 1 + + # update the watchpoint values + if any(name.startswith('W') for name in num2name.values()): + watchvalues = self.all_breakpoints.watchvalues + flat = [] + for n in range(N): + text = '' + name = num2name.get(n, '') + if name.startswith('W'): + text = watchvalues[n] + if text is None: + _, text = self.check_watchpoint_expr(name[1:]) + print 'updating watchpoint value: %s => %s' % ( + name[1:], text) + watchvalues[n] = text + flat.append(text) + extra = '\x00'.join(flat) + self.active.send(Message(CMD_WATCHVALUES, extra=extra)) + self.active.expect_ready() + + self.active.breakpoints_cache = self.all_breakpoints.duplicate() + + def check_watchpoint_expr(self, expr): + self.active.send(Message(CMD_CHECKWATCH, extra=expr)) + msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis) self.active.expect_ready() - self.active.breakpoints_cache = self.all_breakpoints.duplicate() + return msg.arg1, msg.extra def _resume(self, from_time): clone_me = self.paused[from_time] diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -585,14 +585,21 @@ execute_rpy_function(rpy_revdb_commands.rp_funcs[i], cmd, s); } -static void save_state(void) +RPY_EXTERN +bool_t rpy_reverse_db_save_state(void) { - stopped_time = rpy_revdb.stop_point_seen; - stopped_uid = rpy_revdb.unique_id_seen; - rpy_revdb.unique_id_seen = (-1ULL) << 63; + if (stopped_time == 0) { + stopped_time = rpy_revdb.stop_point_seen; + stopped_uid = rpy_revdb.unique_id_seen; + rpy_revdb.unique_id_seen = (-1ULL) << 63; + return 1; + } + else + return 0; } -static void restore_state(void) +RPY_EXTERN +void rpy_reverse_db_restore_state(void) { rpy_revdb.stop_point_seen = stopped_time; rpy_revdb.unique_id_seen = stopped_uid; @@ -604,7 +611,7 @@ void rpy_reverse_db_stop_point(void) { while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { - save_state(); + rpy_reverse_db_save_state(); breakpoint_mode = 0; if (pending_after_forward) { @@ -645,7 +652,7 @@ break; } } - restore_state(); + rpy_reverse_db_restore_state(); } } @@ -744,12 +751,12 @@ } if (rpy_revdb_commands.rp_alloc) { rpy_revdb_t dinfo; - save_state(); + rpy_reverse_db_save_state(); disable_io(&dinfo); if (setjmp(jmp_buf_cancel_execution) == 0) rpy_revdb_commands.rp_alloc(uid, new_object); enable_io(&dinfo); - restore_state(); + rpy_reverse_db_restore_state(); } rpy_revdb.unique_id_break = *future_next_id++; return uid; diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -102,6 +102,12 @@ #define OP_REVDB_TRACK_OBJECT(uid, callback, r) \ rpy_reverse_db_track_object(uid, callback) +#define OP_REVDB_SAVE_STATE(r) \ + r = rpy_reverse_db_save_state() + +#define OP_REVDB_RESTORE_STATE(r) \ + rpy_reverse_db_restore_state() + RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size, const char *file, int line); @@ -114,5 +120,7 @@ RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num); RPY_EXTERN long long rpy_reverse_db_get_value(char value_id); RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object); +RPY_EXTERN bool_t rpy_reverse_db_save_state(void); +RPY_EXTERN void rpy_reverse_db_restore_state(void); /* ------------------------------------------------------------ */ From pypy.commits at gmail.com Wed Jun 22 14:31:20 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 22 Jun 2016 11:31:20 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: updates Message-ID: <576ad978.cf981c0a.4a437.0587@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85336:1a7eac6b97b3 Date: 2016-06-22 18:32 +0200 http://bitbucket.org/pypy/pypy/changeset/1a7eac6b97b3/ Log: updates diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -116,11 +116,11 @@ # ____________________________________________________________ - at specialize.arg(2) -def _change_time(mode, time, callback): - ll_callback = llhelper(_CALLBACK_NOARG_FNPTR, callback) - llop.revdb_change_time(lltype.Void, mode, time, ll_callback) -_CALLBACK_NOARG_FNPTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) +## @specialize.arg(2) +## def _change_time(mode, time, callback): +## ll_callback = llhelper(_CALLBACK_NOARG_FNPTR, callback) +## llop.revdb_change_time(lltype.Void, mode, time, ll_callback) +## _CALLBACK_NOARG_FNPTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) _CALLBACK_GCREF_FNPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Void)) _CMDPTR = rffi.CStructPtr('rpy_revdb_command_s', diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -567,7 +567,6 @@ 'revdb_stop_point': LLOp(), 'revdb_send_answer': LLOp(), - 'revdb_change_time': LLOp(), 'revdb_breakpoint': LLOp(), 'revdb_get_value': LLOp(sideeffects=False), 'revdb_get_unique_id': LLOp(sideeffects=False), diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -31,6 +31,7 @@ last_time = self.pgroup.get_current_time() if last_time != previous_time: print + self.pgroup.update_watch_values() if self.print_extra_pending_info: print self.print_extra_pending_info self.print_extra_pending_info = None @@ -117,16 +118,15 @@ else: return '?????point' - def _bp_new(self, break_at, watchvalue=None): + def _bp_new(self, break_at): b = self.pgroup.edit_breakpoints() new = 1 while new in b.num2name: new += 1 b.num2name[new] = break_at - if watchvalue is not None: - b.watchvalues[new] = watchvalue + if break_at.startswith('W'): + b.watchvalues[new] = '' print "%s %d added" % (self._bp_kind(break_at).capitalize(), new) - return new def cmd_info_breakpoints(self): """List current breakpoints and watchpoints""" @@ -288,5 +288,5 @@ print text print 'Watchpoint not added' else: - print 'Current value:', text - self._bp_new('W' + argument, watchvalue=text) + self._bp_new('W' + argument) + self.pgroup.update_watch_values() diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -75,7 +75,7 @@ return ''.join(pieces) def send(self, msg): - print 'SENT:', self.pid, msg + #print 'SENT:', self.pid, msg binary = struct.pack("iIqqq", msg.cmd, len(msg.extra), msg.arg1, msg.arg2, msg.arg3) self.control_socket.sendall(binary + msg.extra) @@ -85,7 +85,7 @@ cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary) extra = self._recv_all(size) msg = Message(cmd, arg1, arg2, arg3, extra) - print 'RECV:', self.pid, msg + #print 'RECV:', self.pid, msg return msg def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""): @@ -133,7 +133,7 @@ except socket.error: pass - def forward(self, steps, breakpoint_mode, all_breakpoints): + def forward(self, steps, breakpoint_mode): """Move this subprocess forward in time. Returns the Breakpoint or None. """ @@ -149,8 +149,6 @@ break if bkpt is None: bkpt = Breakpoint(msg.arg1, msg.arg3) - all_breakpoints.watchvalues = dict.fromkeys( - all_breakpoints.watchvalues) # set all values to None assert msg.cmd == ANSWER_READY self.update_times(msg) return bkpt @@ -175,7 +173,7 @@ pgroup.all_printed_objects_lst.append(uid) sys.stdout.write('$%d = ' % nid) else: - print >> sys.stderr, "unexpected message %d" % (msg.cmd,) + print >> sys.stderr, "unexpected %r" % (msg,) class ReplayProcessGroup(object): @@ -268,8 +266,7 @@ break assert rel_next_clone >= 0 if rel_next_clone > 0: - bkpt = self.active.forward(rel_next_clone, breakpoint_mode, - self.all_breakpoints) + bkpt = self.active.forward(rel_next_clone, breakpoint_mode) if breakpoint_mode == 'r': latest_bkpt = bkpt or latest_bkpt elif bkpt: @@ -279,8 +276,7 @@ self.paused[self.active.current_time].close() clone = self.active.clone() self.paused[clone.current_time] = clone - bkpt = self.active.forward(steps, breakpoint_mode, - self.all_breakpoints) + bkpt = self.active.forward(steps, breakpoint_mode) if breakpoint_mode == 'r': bkpt = bkpt or latest_bkpt if bkpt: @@ -349,11 +345,6 @@ name = num2name.get(n, '') if name.startswith('W'): text = watchvalues[n] - if text is None: - _, text = self.check_watchpoint_expr(name[1:]) - print 'updating watchpoint value: %s => %s' % ( - name[1:], text) - watchvalues[n] = text flat.append(text) extra = '\x00'.join(flat) self.active.send(Message(CMD_WATCHVALUES, extra=extra)) @@ -361,6 +352,18 @@ self.active.breakpoints_cache = self.all_breakpoints.duplicate() + def update_watch_values(self): + seen = set() + for num, name in self.all_breakpoints.num2name.items(): + if name.startswith('W'): + _, text = self.check_watchpoint_expr(name[1:]) + if text != self.all_breakpoints.watchvalues[num]: + print 'updating watchpoint value: %s => %s' % ( + name[1:], text) + self.all_breakpoints.watchvalues[num] = text + seen.add(num) + assert set(self.all_breakpoints.watchvalues) == seen + def check_watchpoint_expr(self, expr): self.active.send(Message(CMD_CHECKWATCH, extra=expr)) msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis) diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -396,11 +396,18 @@ } } -static void disable_io(rpy_revdb_t *dinfo) +static rpy_revdb_t disabled_state; +static void *disabled_exc[2]; + +static void disable_io(void) { - *dinfo = rpy_revdb; /* save the complete struct */ - dinfo->saved_exc[0] = pypy_g_ExcData.ed_exc_type; - dinfo->saved_exc[1] = pypy_g_ExcData.ed_exc_value; + if (flag_io_disabled) { + fprintf(stderr, "unexpected recursive disable_io()\n"); + exit(1); + } + disabled_state = rpy_revdb; /* save the complete struct */ + disabled_exc[0] = pypy_g_ExcData.ed_exc_type; + disabled_exc[1] = pypy_g_ExcData.ed_exc_value; pypy_g_ExcData.ed_exc_type = NULL; pypy_g_ExcData.ed_exc_value = NULL; rpy_revdb.buf_p = NULL; @@ -408,9 +415,13 @@ flag_io_disabled = 1; } -static void enable_io(rpy_revdb_t *dinfo) +static void enable_io(int restore_breaks) { uint64_t v1, v2; + if (!flag_io_disabled) { + fprintf(stderr, "unexpected enable_io()\n"); + exit(1); + } flag_io_disabled = 0; if (pypy_g_ExcData.ed_exc_type != NULL) { @@ -422,22 +433,23 @@ /* restore the complete struct, with the exception of '*_break' */ v1 = rpy_revdb.stop_point_break; v2 = rpy_revdb.unique_id_break; - rpy_revdb = *dinfo; - rpy_revdb.stop_point_break = v1; - rpy_revdb.unique_id_break = v2; - pypy_g_ExcData.ed_exc_type = dinfo->saved_exc[0]; - pypy_g_ExcData.ed_exc_value = dinfo->saved_exc[1]; + rpy_revdb = disabled_state; + if (!restore_breaks) { + rpy_revdb.stop_point_break = v1; + rpy_revdb.unique_id_break = v2; + } + pypy_g_ExcData.ed_exc_type = disabled_exc[0]; + pypy_g_ExcData.ed_exc_value = disabled_exc[1]; } static void execute_rpy_function(rpy_revdb_command_fn func, rpy_revdb_command_t *cmd, RPyString *extra) { - rpy_revdb_t dinfo; - disable_io(&dinfo); + disable_io(); if (setjmp(jmp_buf_cancel_execution) == 0) func(cmd, extra); - enable_io(&dinfo); + enable_io(0); } static void check_at_end(uint64_t stop_points) @@ -585,13 +597,29 @@ execute_rpy_function(rpy_revdb_commands.rp_funcs[i], cmd, s); } +static void save_state(void) +{ + stopped_time = rpy_revdb.stop_point_seen; + stopped_uid = rpy_revdb.unique_id_seen; + rpy_revdb.unique_id_seen = (-1ULL) << 63; +} + +static void restore_state(void) +{ + rpy_revdb.stop_point_seen = stopped_time; + rpy_revdb.unique_id_seen = stopped_uid; + stopped_time = 0; + stopped_uid = 0; +} + RPY_EXTERN bool_t rpy_reverse_db_save_state(void) { if (stopped_time == 0) { - stopped_time = rpy_revdb.stop_point_seen; - stopped_uid = rpy_revdb.unique_id_seen; - rpy_revdb.unique_id_seen = (-1ULL) << 63; + save_state(); + disable_io(); + rpy_revdb.stop_point_break = 0; + rpy_revdb.unique_id_break = 0; return 1; } else @@ -601,17 +629,15 @@ RPY_EXTERN void rpy_reverse_db_restore_state(void) { - rpy_revdb.stop_point_seen = stopped_time; - rpy_revdb.unique_id_seen = stopped_uid; - stopped_time = 0; - stopped_uid = 0; + enable_io(1); + restore_state(); } RPY_EXTERN void rpy_reverse_db_stop_point(void) { while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { - rpy_reverse_db_save_state(); + save_state(); breakpoint_mode = 0; if (pending_after_forward) { @@ -621,6 +647,8 @@ } else { rpy_revdb_command_t cmd; + if (((int64_t)stopped_uid) < 0) + attach_gdb(); write_answer(ANSWER_READY, stopped_time, stopped_uid, 0); read_sock(&cmd, sizeof(cmd)); @@ -652,7 +680,7 @@ break; } } - rpy_reverse_db_restore_state(); + restore_state(); } } @@ -677,30 +705,6 @@ } RPY_EXTERN -void rpy_reverse_db_change_time(char mode, long long time, - void callback(void)) -{ - switch (mode) { - - case 'f': /* forward */ - if (time < 0) { - fprintf(stderr, "revdb.go_forward(): negative amount of steps\n"); - exit(1); - } - if (stopped_time == 0) { - fprintf(stderr, "revdb.go_forward(): not from a debug command\n"); - exit(1); - } - rpy_revdb.stop_point_break = stopped_time + time; - pending_after_forward = callback; - break; - - default: - abort(); /* unreachable */ - } -} - -RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num) { if (stopped_time != 0) { @@ -750,12 +754,12 @@ exit(1); } if (rpy_revdb_commands.rp_alloc) { - rpy_revdb_t dinfo; - rpy_reverse_db_save_state(); - disable_io(&dinfo); + if (!rpy_reverse_db_save_state()) { + fprintf(stderr, "unexpected recursive unique_id_break\n"); + exit(1); + } if (setjmp(jmp_buf_cancel_execution) == 0) rpy_revdb_commands.rp_alloc(uid, new_object); - enable_io(&dinfo); rpy_reverse_db_restore_state(); } rpy_revdb.unique_id_break = *future_next_id++; diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -18,7 +18,6 @@ char *buf_p, *buf_limit; uint64_t stop_point_seen, stop_point_break; uint64_t unique_id_seen, unique_id_break; - void *saved_exc[2]; } rpy_revdb_t; RPY_EXTERN rpy_revdb_t rpy_revdb; @@ -84,9 +83,6 @@ #define OP_REVDB_SEND_ANSWER(cmd, arg1, arg2, arg3, ll_string, r) \ rpy_reverse_db_send_answer(cmd, arg1, arg2, arg3, ll_string) -#define OP_REVDB_CHANGE_TIME(mode, time, callback, r) \ - rpy_reverse_db_change_time(mode, time, callback) - #define OP_REVDB_BREAKPOINT(num, r) \ rpy_reverse_db_breakpoint(num) @@ -115,8 +111,6 @@ RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2, int64_t arg3, RPyString *extra); RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj); -RPY_EXTERN void rpy_reverse_db_change_time(char mode, long long time, - void callback(void)); RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num); RPY_EXTERN long long rpy_reverse_db_get_value(char value_id); RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object); From pypy.commits at gmail.com Wed Jun 22 15:02:36 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 12:02:36 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix comprehension unpack error Message-ID: <576ae0cc.571b1c0a.bc8c0.10ba@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85337:2060ceb7f13b Date: 2016-06-22 21:01 +0200 http://bitbucket.org/pypy/pypy/changeset/2060ceb7f13b/ Log: Fix comprehension unpack error diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1315,25 +1315,28 @@ return comps def handle_genexp(self, genexp_node): - elt = self.handle_expr(genexp_node.get_child(0)) - if elt.type == syms.star_expr: - self.error("iterable unpacking cannot be used in comprehension", elt) + ch = genexp_node.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) comps = self.comprehension_helper(genexp_node.get_child(1)) return ast.GeneratorExp(elt, comps, genexp_node.get_lineno(), genexp_node.get_column()) def handle_listcomp(self, listcomp_node): - elt = self.handle_expr(listcomp_node.get_child(0)) - if elt.type == syms.star_expr: - self.error("iterable unpacking cannot be used in comprehension", elt) + ch = listcomp_node.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) comps = self.comprehension_helper(listcomp_node.get_child(1)) return ast.ListComp(elt, comps, listcomp_node.get_lineno(), listcomp_node.get_column()) def handle_setcomp(self, set_maker): - elt = self.handle_expr(set_maker.get_child(0)) - if elt.type == syms.star_expr: - self.error("iterable unpacking cannot be used in comprehension", elt) + ch = set_maker.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) comps = self.comprehension_helper(set_maker.get_child(1)) return ast.SetComp(elt, comps, set_maker.get_lineno(), set_maker.get_column()) From pypy.commits at gmail.com Wed Jun 22 16:22:49 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 22 Jun 2016 13:22:49 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: test, fix for PyTuple_Type subclassing Message-ID: <576af399.e643c20a.93272.56c7@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85338:ddf360d978b7 Date: 2016-06-20 23:36 +0300 http://bitbucket.org/pypy/pypy/changeset/ddf360d978b7/ Log: test, fix for PyTuple_Type subclassing diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -620,17 +620,34 @@ (destructor)custom_dealloc, /*tp_dealloc*/ }; +static PyTypeObject TupleLike = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.TupleLike", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; + + static PyObject *size_of_instances(PyObject *self, PyObject *t) { return PyInt_FromLong(((PyTypeObject *)t)->tp_basicsize); } + +static PyObject * is_TupleLike(PyObject *self, PyObject * t) +{ + int tf = t->ob_type == &TupleLike; + Py_DECREF(t); + return PyInt_FromLong(tf); +} + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, {"newCustom", (PyCFunction)newCustom, METH_NOARGS, NULL}, {"size_of_instances", (PyCFunction)size_of_instances, METH_O, NULL}, + {"is_TupleLike", (PyCFunction)is_TupleLike, METH_O, NULL}, {NULL, NULL} /* Sentinel */ }; @@ -680,6 +697,10 @@ if (PyType_Ready(&UnicodeSubtype3) < 0) return; + TupleLike.tp_base = &PyTuple_Type; + if (PyType_Ready(&TupleLike) < 0) + return; + m = Py_InitModule("foo", foo_functions); if (m == NULL) return; @@ -702,4 +723,6 @@ return; if (PyDict_SetItemString(d, "Custom", (PyObject *) &CustomType) < 0) return; + if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) + return; } diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -151,3 +151,8 @@ """), ]) module.run() + + def test_tuple_subclass(self): + module = self.import_module(name='foo') + a = module.TupleLike([1, 2, 3]) + assert module.is_TupleLike(a) diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -108,7 +108,9 @@ "converting a PyTupleObject into a W_TupleObject, " "but found NULLs as items") items_w[i] = w_item - w_obj = space.newtuple(items_w) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_TupleObject, w_type) + w_obj.__init__(items_w) track_reference(space, py_obj, w_obj) return w_obj From pypy.commits at gmail.com Wed Jun 22 16:22:51 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 22 Jun 2016 13:22:51 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: change PyTupleObject.ob_item from PyObject** to PyObject*[], implies tp_itemsize != 0 Message-ID: <576af39b.e7c9c20a.fda45.57be@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85339:1f707f869b48 Date: 2016-06-22 22:43 +0300 http://bitbucket.org/pypy/pypy/changeset/1f707f869b48/ Log: change PyTupleObject.ob_item from PyObject** to PyObject*[], implies tp_itemsize != 0 diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -51,7 +51,7 @@ api._PyTuple_Resize(ar, 10) assert api.PyTuple_Size(ar[0]) == 10 for i in range(3, 10): - rffi.cast(PyTupleObject, py_tuple).c_ob_item[i] = make_ref( + rffi.cast(PyTupleObject, ar[0]).c_ob_item[i] = make_ref( space, space.wrap(42 + i)) w_tuple = from_ref(space, ar[0]) assert space.int_w(space.len(w_tuple)) == 10 diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -2,10 +2,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.debug import fatalerror_notb from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, + build_type_checkers, PyVarObjectFields, cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - make_ref, from_ref, decref, + make_ref, from_ref, decref, pyobj_has_w_obj, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -29,8 +29,8 @@ PyTupleObjectStruct = lltype.ForwardReference() PyTupleObject = lltype.Ptr(PyTupleObjectStruct) ObjectItems = rffi.CArray(PyObject) -PyTupleObjectFields = PyObjectFields + \ - (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +PyTupleObjectFields = PyVarObjectFields + \ + (("ob_item", ObjectItems),) cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) @bootstrap_function @@ -56,14 +56,12 @@ tuple_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_tuple.layout.typedef) - py_obj = typedescr.allocate(space, space.w_tuple) + py_obj = typedescr.allocate(space, space.w_tuple, length) py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_ob_item = lltype.malloc(ObjectItems, length, - flavor='raw', zero=True, - add_memory_pressure=True) - py_tup.c_ob_size = length - return py_tup + p = py_tup.c_ob_item + for i in range(py_tup.c_ob_size): + p[i] = lltype.nullptr(PyObject.TO) + return py_obj def tuple_attach(space, py_obj, w_obj): """ @@ -71,23 +69,22 @@ buffer must not be modified. """ items_w = space.fixedview(w_obj) - l = len(items_w) - p = lltype.malloc(ObjectItems, l, flavor='raw', - add_memory_pressure=True) + py_tup = rffi.cast(PyTupleObject, py_obj) + length = len(items_w) + if py_tup.c_ob_size < length: + raise oefmt(space.w_ValueError, + "tuple_attach called on object with ob_size %d but trying to store %d", + py_tup.c_ob_size, length) i = 0 try: - while i < l: - p[i] = make_ref(space, items_w[i]) + while i < length: + py_tup.c_ob_item[i] = make_ref(space, items_w[i]) i += 1 except: while i > 0: i -= 1 - decref(space, p[i]) - lltype.free(p, flavor='raw') + decref(space, py_tup.c_ob_item[i]) raise - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_ob_size = l - py_tup.c_ob_item = p def tuple_realize(space, py_obj): """ @@ -101,7 +98,9 @@ p = py_tup.c_ob_item items_w = [None] * l for i in range(l): - w_item = from_ref(space, p[i]) + w_item = None + if p[i]: + w_item = from_ref(space, p[i]) if w_item is None: fatalerror_notb( "Fatal error in cpyext, CPython compatibility layer: " @@ -120,18 +119,17 @@ """ py_tup = rffi.cast(PyTupleObject, py_obj) p = py_tup.c_ob_item - if p: - for i in range(py_tup.c_ob_size): + for i in range(py_tup.c_ob_size): + if p[i] and p[i].c_ob_refcnt > 0: decref(space, p[i]) - lltype.free(p, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + while py_obj.c_ob_refcnt > 0: + decref(space, py_obj) #_______________________________________________________________________ @cpython_api([Py_ssize_t], PyObject, result_is_ll=True) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return new_empty_tuple(space, size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PyTuple_SetItem(space, ref, index, py_obj): @@ -187,25 +185,23 @@ ref = p_ref[0] if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - ref = rffi.cast(PyTupleObject, ref) - oldsize = ref.c_ob_size - oldp = ref.c_ob_item - newp = lltype.malloc(ObjectItems, newsize, zero=True, flavor='raw', - add_memory_pressure=True) + oldref = rffi.cast(PyTupleObject, ref) + oldsize = oldref.c_ob_size + p_ref[0] = new_empty_tuple(space, newsize) + newref = rffi.cast(PyTupleObject, p_ref[0]) try: if oldsize < newsize: to_cp = oldsize else: to_cp = newsize for i in range(to_cp): - newp[i] = oldp[i] + newref.c_ob_item[i] = oldref.c_ob_item[i] except: - lltype.free(newp, flavor='raw') + decref(space, p_ref[0]) + p_ref[0] = lltype.nullptr(PyObject.TO) raise - ref.c_ob_item = newp - ref.c_ob_size = newsize - lltype.free(oldp, flavor='raw') - # in this version, p_ref[0] never needs to be updated + finally: + decref(space, ref) return 0 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -659,6 +659,8 @@ subtype_dealloc.api_func.get_wrapper(space)) if space.is_w(w_type, space.w_str): pto.c_tp_itemsize = 1 + elif space.is_w(w_type, space.w_tuple): + pto.c_tp_itemsize = rffi.sizeof(PyObject) # buffer protocol setup_buffer_procs(space, w_type, pto) From pypy.commits at gmail.com Wed Jun 22 16:22:53 2016 From: pypy.commits at gmail.com (mattip) Date: Wed, 22 Jun 2016 13:22:53 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: make sure itemsize is inherited Message-ID: <576af39d.e643c20a.93272.56cd@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85340:18b7886e7acd Date: 2016-06-22 23:21 +0300 http://bitbucket.org/pypy/pypy/changeset/18b7886e7acd/ Log: make sure itemsize is inherited diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -637,7 +637,8 @@ static PyObject * is_TupleLike(PyObject *self, PyObject * t) { int tf = t->ob_type == &TupleLike; - Py_DECREF(t); + if (t->ob_type->tp_itemsize == 0) + return PyInt_FromLong(-1); return PyInt_FromLong(tf); } diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -155,4 +155,4 @@ def test_tuple_subclass(self): module = self.import_module(name='foo') a = module.TupleLike([1, 2, 3]) - assert module.is_TupleLike(a) + assert module.is_TupleLike(a) == 1 diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -371,6 +371,8 @@ # (minimally, if tp_basicsize is zero we copy it from the base) if not pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize + if pto.c_tp_itemsize < base_pto.c_tp_itemsize: + pto.c_tp_itemsize = base_pto.c_tp_itemsize flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) From pypy.commits at gmail.com Wed Jun 22 16:50:10 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 13:50:10 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Forgot self at two function calls Message-ID: <576afa02.e643c20a.93272.5f29@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85341:87c42f87967a Date: 2016-06-22 21:08 +0200 http://bitbucket.org/pypy/pypy/changeset/87c42f87967a/ Log: Forgot self at two function calls diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1220,7 +1220,7 @@ (n_maker_children > 1 and maker.get_child(1).type == tokens.COMMA)): # a set display - return handle_setdisplay(maker) + return self.handle_setdisplay(maker) elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for: # a set comprehension return self.handle_setcomp(maker) @@ -1234,7 +1234,7 @@ return self.handle_dictcomp(maker) else: # a dictionary display - return handle_dictdisplay(maker) + return self.handle_dictdisplay(maker) else: raise AssertionError("unknown atom") From pypy.commits at gmail.com Wed Jun 22 16:50:12 2016 From: pypy.commits at gmail.com (raffael_t) Date: Wed, 22 Jun 2016 13:50:12 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Delete unnecessary code and comments Message-ID: <576afa04.89dec20a.d8485.6022@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85342:1710eedf630e Date: 2016-06-22 22:49 +0200 http://bitbucket.org/pypy/pypy/changeset/1710eedf630e/ Log: Delete unnecessary code and comments diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1145,13 +1145,13 @@ if isinstance(elt, ast.Starred): # A star-arg. If we've seen positional arguments, # pack the positional arguments into a tuple. - if nseen != 0: + if nseen: self.emit_op_arg(ops.BUILD_TUPLE, nseen) nseen = 0 nsubargs += 1 - elt.value.walkabout(self) #self.visit(elt.value) # check orig, elt->v.Starred.value + elt.value.walkabout(self) nsubargs += 1 - elif nsubargs != 0: + elif nsubargs: # We've seen star-args already, so we # count towards items-to-pack-into-tuple. elt.walkabout(self) @@ -1161,11 +1161,11 @@ # are left on the stack. elt.walkabout(self) n += 1 - if nseen != 0: + if nseen: # Pack up any trailing positional arguments. self.emit_op_arg(ops.BUILD_TUPLE, nseen) nsubargs += 1 - if nsubargs != 0: + if nsubargs: call_type |= 1 if nsubargs > 1: # If we ended up with more than one stararg, we need @@ -1178,13 +1178,13 @@ for kw in keywords: if kw.arg is None: # A keyword argument unpacking. - if nseen != 0: + if nseen: self.emit_op_arg(ops.BUILD_MAP, nseen) nseen = 0 nsubkwargs += 1 - kw.value.walkabout(self) #self.visit_sequence(kw.value) # probably wrong, elt->v.Starred.value + kw.value.walkabout(self) nsubkwargs += 1 - elif nsubkwargs != 0: + elif nsubkwargs: # A keyword argument and we already have a dict. self.load_const(kw.arg) kw.value.walkabout(self) @@ -1193,11 +1193,11 @@ # keyword argument kw.walkabout(self) nkw += 1 - if nseen != 0: + if nseen: # Pack up any trailing keyword arguments. self.emit_op_arg(ops.BUILD_MAP,nseen) nsubkwargs += 1 - if nsubargs != 0: + if nsubargs: call_type |= 2 if nsubkwargs > 1: # Pack it all up From pypy.commits at gmail.com Thu Jun 23 03:57:12 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 00:57:12 -0700 (PDT) Subject: [pypy-commit] pypy default: Merged in markrwilliams/pypy/save_socket_errno (pull request #458) Message-ID: <576b9658.c5301c0a.4b84b.ffffd824@mx.google.com> Author: Armin Rigo Branch: Changeset: r85347:aa8fac09d527 Date: 2016-06-23 09:55 +0200 http://bitbucket.org/pypy/pypy/changeset/aa8fac09d527/ Log: Merged in markrwilliams/pypy/save_socket_errno (pull request #458) rffi's socket(2) wrapper did not preserve errno. diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 03:57:25 2016 From: pypy.commits at gmail.com (markrwilliams) Date: Thu, 23 Jun 2016 00:57:25 -0700 (PDT) Subject: [pypy-commit] pypy save_socket_errno: rffi's socket(2) wrapper did not preserve errno. Message-ID: <576b9665.094ac20a.d2757.07e5@mx.google.com> Author: Mark Williams Branch: save_socket_errno Changeset: r85343:5922e2ab336d Date: 2016-06-22 22:01 -0700 http://bitbucket.org/pypy/pypy/changeset/5922e2ab336d/ Log: rffi's socket(2) wrapper did not preserve errno. Preserve errno, like the other socket system call wrappers, and add a test to ensure the correct errno is raised upon socket failure. diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -1,4 +1,4 @@ -import py, errno, sys +import py, pytest, errno, sys from rpython.rlib import rsocket from rpython.rlib.rsocket import * import socket as cpy_socket @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + pytest.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = pytest.raises(CSocketError, + RSocket, family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + # TODO: windows?? + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 03:57:27 2016 From: pypy.commits at gmail.com (markrwilliams) Date: Thu, 23 Jun 2016 00:57:27 -0700 (PDT) Subject: [pypy-commit] pypy save_socket_errno: Remove spurious TODO Message-ID: <576b9667.4f8d1c0a.536c3.ffffdf53@mx.google.com> Author: Mark Williams Branch: save_socket_errno Changeset: r85344:e67817facadb Date: 2016-06-22 22:05 -0700 http://bitbucket.org/pypy/pypy/changeset/e67817facadb/ Log: Remove spurious TODO diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -599,5 +599,4 @@ e = pytest.raises(CSocketError, RSocket, family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) - # TODO: windows?? assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 03:57:29 2016 From: pypy.commits at gmail.com (markrwilliams) Date: Thu, 23 Jun 2016 00:57:29 -0700 (PDT) Subject: [pypy-commit] pypy save_socket_errno: Remove spurious pytest import Message-ID: <576b9669.571b1c0a.bc8c0.ffffda6e@mx.google.com> Author: Mark Williams Branch: save_socket_errno Changeset: r85345:5f0b96606d72 Date: 2016-06-22 22:07 -0700 http://bitbucket.org/pypy/pypy/changeset/5f0b96606d72/ Log: Remove spurious pytest import diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -1,4 +1,4 @@ -import py, pytest, errno, sys +import py, errno, sys from rpython.rlib import rsocket from rpython.rlib.rsocket import * import socket as cpy_socket From pypy.commits at gmail.com Thu Jun 23 03:57:30 2016 From: pypy.commits at gmail.com (markrwilliams) Date: Thu, 23 Jun 2016 00:57:30 -0700 (PDT) Subject: [pypy-commit] pypy save_socket_errno: Remove spurious pytest import and ensure test runs correctly Message-ID: <576b966a.07ecc20a.53fa1.0c9d@mx.google.com> Author: Mark Williams Branch: save_socket_errno Changeset: r85346:decbd0844046 Date: 2016-06-22 22:10 -0700 http://bitbucket.org/pypy/pypy/changeset/decbd0844046/ Log: Remove spurious pytest import and ensure test runs correctly diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -593,10 +593,11 @@ def test_socket_saves_errno(tmpdir): # ensure errno is set to a known value... unconnected_sock = RSocket() - pytest.raises(CSocketError, unconnected_sock.recv, 1024) + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) # ...which is ENOTCONN assert e.value.errno == errno.ENOTCONN - e = pytest.raises(CSocketError, - RSocket, family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 04:10:10 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 01:10:10 -0700 (PDT) Subject: [pypy-commit] pypy default: Document branch by markrwilliams Message-ID: <576b9962.e409c20a.37692.0cd9@mx.google.com> Author: Armin Rigo Branch: Changeset: r85348:5ca150c78c39 Date: 2016-06-23 10:11 +0200 http://bitbucket.org/pypy/pypy/changeset/5ca150c78c39/ Log: Document branch by markrwilliams diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,9 @@ Allow rw access to the char* returned from PyString_AS_STRING, also refactor PyStringObject to look like cpython's and allow subclassing PyString_Type and PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. From pypy.commits at gmail.com Thu Jun 23 04:18:19 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 01:18:19 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: tweaks Message-ID: <576b9b4b.cc9d1c0a.c84f2.ffffde60@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85349:f48e08e15ad8 Date: 2016-06-23 10:19 +0200 http://bitbucket.org/pypy/pypy/changeset/f48e08e15ad8/ Log: tweaks diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -106,11 +106,11 @@ ll_callback = llhelper(_CALLBACK_GCREF_FNPTR, callback) llop.revdb_track_object(lltype.Void, unique_id, ll_callback) -def save_state(): - return llop.revdb_save_state(lltype.Bool) +def watch_save_state(): + return llop.revdb_watch_save_state(lltype.Bool) -def restore_state(): - llop.revdb_restore_state(lltype.Void) +def watch_restore_state(any_watch_point): + llop.revdb_watch_restore_state(lltype.Void, any_watch_point) # ____________________________________________________________ diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -570,8 +570,8 @@ 'revdb_breakpoint': LLOp(), 'revdb_get_value': LLOp(sideeffects=False), 'revdb_get_unique_id': LLOp(sideeffects=False), - 'revdb_save_state': LLOp(), - 'revdb_restore_state': LLOp(), + 'revdb_watch_save_state': LLOp(), + 'revdb_watch_restore_state': LLOp(), } # ***** Run test_lloperation after changes. ***** diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -126,6 +126,8 @@ b.num2name[new] = break_at if break_at.startswith('W'): b.watchvalues[new] = '' + nids = map(int, r_dollar_num.findall(break_at[1:])) + b.watchdollars[new] = nids print "%s %d added" % (self._bp_kind(break_at).capitalize(), new) def cmd_info_breakpoints(self): @@ -275,6 +277,7 @@ else: name = b.num2name.pop(arg) b.watchvalues.pop(arg, '') + b.watchdollars.pop(arg, '') kind = self._bp_kind(name) print "%s %d deleted: %s" % (kind.capitalize(), arg, name[1:]) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -20,6 +20,7 @@ def __init__(self): self.num2name = {} # {small number: break/watchpoint} self.watchvalues = {} # {small number: resulting text} + self.watchdollars = {} # {small number: [nids]} self.stack_depth = 0 # breaks if the depth becomes lower than this def __repr__(self): @@ -75,7 +76,7 @@ return ''.join(pieces) def send(self, msg): - #print 'SENT:', self.pid, msg + print 'SENT:', self.pid, msg binary = struct.pack("iIqqq", msg.cmd, len(msg.extra), msg.arg1, msg.arg2, msg.arg3) self.control_socket.sendall(binary + msg.extra) @@ -85,7 +86,7 @@ cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary) extra = self._recv_all(size) msg = Message(cmd, arg1, arg2, arg3, extra) - #print 'RECV:', self.pid, msg + print 'RECV:', self.pid, msg return msg def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""): diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -552,6 +552,7 @@ last_recorded_breakpoint_loc = 0; pending_after_forward = &answer_recorded_breakpoint; } + rpy_revdb.watch_enabled = (breakpoint_mode != 'i'); } static void command_future_ids(rpy_revdb_command_t *cmd, char *extra) @@ -602,6 +603,7 @@ stopped_time = rpy_revdb.stop_point_seen; stopped_uid = rpy_revdb.unique_id_seen; rpy_revdb.unique_id_seen = (-1ULL) << 63; + rpy_revdb.watch_enabled = 0; } static void restore_state(void) @@ -613,24 +615,25 @@ } RPY_EXTERN -bool_t rpy_reverse_db_save_state(void) +void rpy_reverse_db_watch_save_state(void) { - if (stopped_time == 0) { - save_state(); - disable_io(); - rpy_revdb.stop_point_break = 0; - rpy_revdb.unique_id_break = 0; - return 1; + if (stopped_time != 0) { + fprintf(stderr, "unexpected recursive watch_save_state\n"); + exit(1); } - else - return 0; + save_state(); + disable_io(); + rpy_revdb.stop_point_break = 0; + rpy_revdb.unique_id_break = 0; } RPY_EXTERN -void rpy_reverse_db_restore_state(void) +void rpy_reverse_db_watch_restore_state(bool_t any_watch_point) { enable_io(1); restore_state(); + assert(!rpy_revdb.watch_enabled); + rpy_revdb.watch_enabled = any_watch_point; } RPY_EXTERN @@ -754,13 +757,11 @@ exit(1); } if (rpy_revdb_commands.rp_alloc) { - if (!rpy_reverse_db_save_state()) { - fprintf(stderr, "unexpected recursive unique_id_break\n"); - exit(1); - } + bool_t watch_enabled = rpy_revdb.watch_enabled; + rpy_reverse_db_watch_save_state(); if (setjmp(jmp_buf_cancel_execution) == 0) rpy_revdb_commands.rp_alloc(uid, new_object); - rpy_reverse_db_restore_state(); + rpy_reverse_db_watch_restore_state(watch_enabled); } rpy_revdb.unique_id_break = *future_next_id++; return uid; diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -15,6 +15,7 @@ #define RPY_RDB_REPLAY rpy_revdb.replay #define RPY_RDB_DYNAMIC_REPLAY #endif + bool_t watch_enabled; char *buf_p, *buf_limit; uint64_t stop_point_seen, stop_point_break; uint64_t unique_id_seen, unique_id_break; @@ -98,11 +99,13 @@ #define OP_REVDB_TRACK_OBJECT(uid, callback, r) \ rpy_reverse_db_track_object(uid, callback) -#define OP_REVDB_SAVE_STATE(r) \ - r = rpy_reverse_db_save_state() +#define OP_REVDB_WATCH_SAVE_STATE(r) do { \ + r = rpy_revdb.watch_enabled; \ + if (r) rpy_reverse_db_watch_save_state(); \ + } while (0) -#define OP_REVDB_RESTORE_STATE(r) \ - rpy_reverse_db_restore_state() +#define OP_REVDB_WATCH_RESTORE_STATE(any_watch_point, r) \ + rpy_reverse_db_watch_restore_state(any_watch_point) RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size, @@ -114,7 +117,7 @@ RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num); RPY_EXTERN long long rpy_reverse_db_get_value(char value_id); RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object); -RPY_EXTERN bool_t rpy_reverse_db_save_state(void); -RPY_EXTERN void rpy_reverse_db_restore_state(void); +RPY_EXTERN void rpy_reverse_db_watch_save_state(void); +RPY_EXTERN void rpy_reverse_db_watch_restore_state(bool_t any_watch_point); /* ------------------------------------------------------------ */ From pypy.commits at gmail.com Thu Jun 23 05:06:00 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 02:06:00 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <576ba678.e409c20a.37692.22f7@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85350:6137b7409336 Date: 2016-06-23 11:06 +0200 http://bitbucket.org/pypy/pypy/changeset/6137b7409336/ Log: in-progress diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -118,7 +118,7 @@ else: return '?????point' - def _bp_new(self, break_at): + def _bp_new(self, break_at, nids=None): b = self.pgroup.edit_breakpoints() new = 1 while new in b.num2name: @@ -126,8 +126,8 @@ b.num2name[new] = break_at if break_at.startswith('W'): b.watchvalues[new] = '' - nids = map(int, r_dollar_num.findall(break_at[1:])) - b.watchdollars[new] = nids + if nids: + b.watchuids[new] = self.pgroup.nids_to_uids(nids) print "%s %d added" % (self._bp_kind(break_at).capitalize(), new) def cmd_info_breakpoints(self): @@ -277,7 +277,7 @@ else: name = b.num2name.pop(arg) b.watchvalues.pop(arg, '') - b.watchdollars.pop(arg, '') + b.watchuids.pop(arg, '') kind = self._bp_kind(name) print "%s %d deleted: %s" % (kind.capitalize(), arg, name[1:]) @@ -286,10 +286,11 @@ if not argument: print "Watch what?" return - ok_flag, text = self.pgroup.check_watchpoint_expr(argument) + nids = map(int, r_dollar_num.findall(argument)) + ok_flag, text = self.pgroup.check_watchpoint_expr(argument, nids) if not ok_flag: print text print 'Watchpoint not added' else: - self._bp_new('W' + argument) + self._bp_new('W' + argument, nids=nids) self.pgroup.update_watch_values() diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -20,7 +20,7 @@ def __init__(self): self.num2name = {} # {small number: break/watchpoint} self.watchvalues = {} # {small number: resulting text} - self.watchdollars = {} # {small number: [nids]} + self.watchuids = {} # {small number: [uid...]} self.stack_depth = 0 # breaks if the depth becomes lower than this def __repr__(self): @@ -319,8 +319,13 @@ search_start_time -= time_range_to_search * 3 def update_breakpoints(self): + if self.all_breakpoints.watchuids: + uids = set() + uids.update(*self.all_breakpoints.watchuids.values()) + self.attach_printed_objects(uids, watch_env=True) + cmp = self.all_breakpoints.compare(self.active.breakpoints_cache) - print 'compare:', cmp, self.all_breakpoints.watchvalues + #print 'compare:', cmp, self.all_breakpoints.watchvalues if cmp == 2: return # up-to-date @@ -365,7 +370,10 @@ seen.add(num) assert set(self.all_breakpoints.watchvalues) == seen - def check_watchpoint_expr(self, expr): + def check_watchpoint_expr(self, expr, nids=None): + if nids: + uids = self.nids_to_uids(nids) + self.attach_printed_objects(uids, watch_env=True) self.active.send(Message(CMD_CHECKWATCH, extra=expr)) msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis) self.active.expect_ready() @@ -385,9 +393,9 @@ target_time = 1 if target_time > self.total_stop_points: target_time = self.total_stop_points - self._resume(max(time for time in self.paused if time <= target_time)) - self.go_forward(target_time - self.get_current_time(), - breakpoint_mode='i') + uids = set() + uids.update(*self.all_breakpoints.watchuids.values()) + self.ensure_printed_objects(uids, forced_time = target_time) def close(self): """Close all subprocesses. @@ -395,12 +403,19 @@ for subp in [self.active] + self.paused.values(): subp.close() - def ensure_printed_objects(self, uids): + def ensure_printed_objects(self, uids, forced_time=None): """Ensure that all the given unique_ids are loaded in the active child, if necessary by forking another child from earlier. """ - initial_time = self.get_current_time() - child = self.active + if forced_time is None: + initial_time = self.get_current_time() + child = self.active + else: + initial_time = forced_time + stop_time = max(time for time in self.paused + if time <= initial_time) + child = self.paused[stop_time] + while True: uid_limit = child.currently_created_objects missing_uids = [uid for uid in uids @@ -420,39 +435,48 @@ assert not future_uids else: self._resume(stop_time) - future_uids.sort() - pack_uids = [struct.pack('q', uid) for uid in future_uids] - self.active.send(Message(CMD_FUTUREIDS, extra=''.join(pack_uids))) - self.active.expect_ready() - self.active.printed_objects = ( - self.active.printed_objects.union(future_uids)) + if future_uids: + future_uids.sort() + pack_uids = [struct.pack('q', uid) for uid in future_uids] + pack_uids = ''.join(pack_uids) + self.active.send(Message(CMD_FUTUREIDS, extra=pack_uids)) + self.active.expect_ready() + self.active.printed_objects = ( + self.active.printed_objects.union(future_uids)) self.go_forward(initial_time - self.get_current_time(), breakpoint_mode='i') assert self.active.printed_objects.issuperset(uids) + def nids_to_uids(self, nids): + uids = [] + for nid in set(nids): + try: + uid = self.all_printed_objects_lst[nid] + except IndexError: + continue + if uid >= self.get_currently_created_objects(): + print >> sys.stderr, ( + "note: '$%d' refers to an object that is " + "only created later in time" % nid) + uids.append(uid) + return uids + + def attach_printed_objects(self, uids, watch_env=False): + for uid in uids: + nid = self.all_printed_objects[uid] + self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env))) + self.active.expect_ready() + def print_cmd(self, expression, nids=[]): """Print an expression. """ uids = [] if nids: - for nid in set(nids): - try: - uid = self.all_printed_objects_lst[nid] - except IndexError: - continue - if uid >= self.get_currently_created_objects(): - print >> sys.stderr, ( - "note: '$%d' refers to an object that is " - "only created later in time" % nid) - continue - uids.append(uid) + uids = self.nids_to_uids(nids) self.ensure_printed_objects(uids) # self.active.tainted = True - for uid in uids: - nid = self.all_printed_objects[uid] - self.active.send(Message(CMD_ATTACHID, nid, uid)) - self.active.expect_ready() + self.attach_printed_objects(uids) self.active.send(Message(CMD_PRINT, extra=expression)) self.active.print_text_answer(pgroup=self) From pypy.commits at gmail.com Thu Jun 23 05:25:49 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 02:25:49 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: fixes Message-ID: <576bab1d.e5acc20a.76713.3175@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85351:583e7d930dbd Date: 2016-06-23 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/583e7d930dbd/ Log: fixes diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -294,8 +294,12 @@ if self.all_breakpoints.is_empty() or ignore_breakpoints: self.jump_in_time(initial_time - steps) else: + if self.all_breakpoints.watchvalues: + first_steps = 97 # use smaller steps, because that's costly + else: + first_steps = 957 self._backward_search_forward( - search_start_time = initial_time - 957, + search_start_time = initial_time - first_steps, search_stop_time = initial_time - 1, search_go_on_until_time = initial_time - steps) @@ -318,12 +322,14 @@ search_stop_time = search_start_time search_start_time -= time_range_to_search * 3 - def update_breakpoints(self): + def _update_watchpoints_uids(self): if self.all_breakpoints.watchuids: uids = set() uids.update(*self.all_breakpoints.watchuids.values()) self.attach_printed_objects(uids, watch_env=True) + def update_breakpoints(self): + self._update_watchpoints_uids() cmp = self.all_breakpoints.compare(self.active.breakpoints_cache) #print 'compare:', cmp, self.all_breakpoints.watchvalues if cmp == 2: @@ -359,6 +365,7 @@ self.active.breakpoints_cache = self.all_breakpoints.duplicate() def update_watch_values(self): + self._update_watchpoints_uids() seen = set() for num, name in self.all_breakpoints.num2name.items(): if name.startswith('W'): @@ -447,21 +454,22 @@ breakpoint_mode='i') assert self.active.printed_objects.issuperset(uids) - def nids_to_uids(self, nids): + def nids_to_uids(self, nids, skip_futures=False): uids = [] for nid in set(nids): try: uid = self.all_printed_objects_lst[nid] except IndexError: continue - if uid >= self.get_currently_created_objects(): + if skip_futures and uid >= self.get_currently_created_objects(): print >> sys.stderr, ( "note: '$%d' refers to an object that is " "only created later in time" % nid) + continue uids.append(uid) return uids - def attach_printed_objects(self, uids, watch_env=False): + def attach_printed_objects(self, uids, watch_env): for uid in uids: nid = self.all_printed_objects[uid] self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env))) @@ -472,11 +480,11 @@ """ uids = [] if nids: - uids = self.nids_to_uids(nids) + uids = self.nids_to_uids(nids, skip_futures=True) self.ensure_printed_objects(uids) # self.active.tainted = True - self.attach_printed_objects(uids) + self.attach_printed_objects(uids, watch_env=False) self.active.send(Message(CMD_PRINT, extra=expression)) self.active.print_text_answer(pgroup=self) From pypy.commits at gmail.com Thu Jun 23 10:32:03 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 23 Jun 2016 07:32:03 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: merge default into branch Message-ID: <576bf2e3.4ccf1c0a.9d988.30a9@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85352:ea8fefb608c9 Date: 2016-06-23 17:30 +0300 http://bitbucket.org/pypy/pypy/changeset/ea8fefb608c9/ Log: merge default into branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,9 @@ Allow rw access to the char* returned from PyString_AS_STRING, also refactor PyStringObject to look like cpython's and allow subclassing PyString_Type and PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -800,6 +800,21 @@ pypy_debug_catch_fatal_exception() assert False +def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid): + from rpython.rlib import rgil + # see "Handling of the GIL" above + assert cpyext_glob_tid_ptr[0] == 0 + if pygilstate_release: + from pypy.module.cpyext import pystate + unlock = (gilstate == pystate.PyGILState_UNLOCKED) + else: + unlock = gil_release or _gil_auto + if unlock: + rgil.release() + else: + cpyext_glob_tid_ptr[0] = tid + + def make_wrapper_second_level(space, argtypesw, restype, result_kind, error_value, gil): from rpython.rlib import rgil @@ -827,6 +842,7 @@ def wrapper_second_level(callable, pname, *args): from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj from pypy.module.cpyext.pyobject import as_pyobj + from pypy.module.cpyext import pystate # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -839,7 +855,6 @@ rgil.acquire() assert cpyext_glob_tid_ptr[0] == 0 elif pygilstate_ensure: - from pypy.module.cpyext import pystate if cpyext_glob_tid_ptr[0] == tid: cpyext_glob_tid_ptr[0] = 0 args += (pystate.PyGILState_LOCKED,) @@ -850,6 +865,10 @@ if cpyext_glob_tid_ptr[0] != tid: no_gil_error(pname) cpyext_glob_tid_ptr[0] = 0 + if pygilstate_release: + gilstate = rffi.cast(lltype.Signed, args[-1]) + else: + gilstate = pystate.PyGILState_IGNORE rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -919,24 +938,13 @@ except Exception as e: unexpected_exception(pname, e, tb) + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return fatal_value assert lltype.typeOf(retval) == restype rffi.stackcounter.stacks_counter -= 1 - # see "Handling of the GIL" above - assert cpyext_glob_tid_ptr[0] == 0 - if pygilstate_release: - from pypy.module.cpyext import pystate - arg = rffi.cast(lltype.Signed, args[-1]) - unlock = (arg == pystate.PyGILState_UNLOCKED) - else: - unlock = gil_release or _gil_auto - if unlock: - rgil.release() - else: - cpyext_glob_tid_ptr[0] = tid - + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return retval wrapper_second_level._dont_inline_ = True diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -207,6 +207,7 @@ PyGILState_STATE = rffi.INT PyGILState_LOCKED = 0 PyGILState_UNLOCKED = 1 +PyGILState_IGNORE = 2 ExecutionContext.cpyext_gilstate_counter_noleave = 0 diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,4 +1,4 @@ -import py +import os import pytest def pytest_configure(config): @@ -21,3 +21,14 @@ def pytest_funcarg__api(request): return request.cls.api +if os.name == 'nt': + @pytest.yield_fixture(autouse=True, scope='session') + def prevent_dialog_box(): + """Do not open dreaded dialog box on segfault on Windows""" + import ctypes + SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN + old_err_mode = ctypes.windll.kernel32.GetErrorMode() + new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX + ctypes.windll.kernel32.SetErrorMode(new_err_mode) + yield + ctypes.windll.kernel32.SetErrorMode(old_err_mode) diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/support.py @@ -0,0 +1,68 @@ +import os +import py +from sys import platform + +if os.name != 'nt': + so_ext = 'so' +else: + so_ext = 'dll' + +def c_compile(cfilenames, outputfilename, + compile_extra=None, link_extra=None, + include_dirs=None, libraries=None, library_dirs=None): + compile_extra = compile_extra or [] + link_extra = link_extra or [] + include_dirs = include_dirs or [] + libraries = libraries or [] + library_dirs = library_dirs or [] + if platform == 'win32': + link_extra = link_extra + ['/DEBUG'] # generate .pdb file + if platform == 'darwin': + # support Fink & Darwinports + for s in ('/sw/', '/opt/local/'): + if (s + 'include' not in include_dirs + and os.path.exists(s + 'include')): + include_dirs.append(s + 'include') + if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): + library_dirs.append(s + 'lib') + + outputfilename = py.path.local(outputfilename).new(ext=so_ext) + saved_environ = os.environ.copy() + try: + _build( + cfilenames, outputfilename, + compile_extra, link_extra, + include_dirs, libraries, library_dirs) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(cfilenames, outputfilename, compile_extra, link_extra, + include_dirs, libraries, library_dirs): + from distutils.ccompiler import new_compiler + from distutils import sysconfig + compiler = new_compiler(force=1) + sysconfig.customize_compiler(compiler) # XXX + objects = [] + for cfile in cfilenames: + cfile = py.path.local(cfile) + old = cfile.dirpath().chdir() + try: + res = compiler.compile([cfile.basename], + include_dirs=include_dirs, extra_preargs=compile_extra) + assert len(res) == 1 + cobjfile = py.path.local(res[0]) + assert cobjfile.check() + objects.append(str(cobjfile)) + finally: + old.chdir() + + compiler.link_shared_object( + objects, str(outputfilename), + libraries=libraries, + extra_preargs=link_extra, + library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestStringObject(AppTestCpythonExtensionBase): def test_basic(self): module = self.import_extension('foo', [ @@ -31,7 +32,7 @@ if(s->ob_type->tp_basicsize != expected_size) { printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); + (long)s->ob_type->tp_basicsize); result = 0; } Py_DECREF(s); @@ -53,7 +54,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyByteArray_FromStringAndSize(NULL, 4); if (s == NULL) @@ -84,11 +84,10 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); - memcpy(PyByteArray_AS_STRING(base), "works", 6); + memcpy(PyByteArray_AS_STRING(base), "works", 6); Py_INCREF(base); return base; """), @@ -115,6 +114,7 @@ assert s == 'test' def test_manipulations(self): + import sys module = self.import_extension('foo', [ ("bytearray_from_string", "METH_VARARGS", ''' @@ -141,9 +141,9 @@ ("concat", "METH_VARARGS", """ PyObject * ret, *right, *left; - PyObject *ba1, *ba2; + PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -157,7 +157,9 @@ """)]) assert module.bytearray_from_string("huheduwe") == "huhe" assert module.str_from_bytearray(bytearray('abc')) == 'abc' - raises(ValueError, module.str_from_bytearray, 4.0) + if '__pypy__' in sys.builtin_module_names: + # CPython only makes an assert. + raises(ValueError, module.str_from_bytearray, 4.0) ret = module.concat('abc', 'def') assert ret == 'abcdef' assert not isinstance(ret, str) @@ -171,9 +173,9 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } - + ba = PyByteArray_FromObject(obj); if (ba == NULL) return NULL; @@ -187,7 +189,7 @@ { printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); return NULL; - } + } return ba; ''' )]) diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -29,7 +29,7 @@ size_t expected_size; result = PyString_Size(s); - + #ifdef PYPY_VERSION expected_size = 48; #elif defined Py_DEBUG @@ -39,7 +39,7 @@ #endif if(s->ob_type->tp_basicsize != expected_size) { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); + printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); result = 0; } Py_DECREF(s); @@ -48,7 +48,7 @@ ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyString_Size(f); + PyString_Size(f); Py_DECREF(f); return NULL; @@ -71,7 +71,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyString_FromStringAndSize(NULL, 4); if (s == NULL) @@ -99,7 +98,6 @@ PyObject *base; PyTypeObject * type; PyStringObject *obj; - char * p_str; base = PyString_FromString("test"); if (PyString_GET_SIZE(base) != 4) return PyLong_FromLong(-PyString_GET_SIZE(base)); @@ -117,7 +115,6 @@ ('alloc_rw', "METH_NOARGS", ''' PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); - char * buf = PyString_AS_STRING(obj); memcpy(PyString_AS_STRING(obj), "works", 6); return (PyObject*)obj; '''), @@ -320,17 +317,17 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyStringObject*)obj)->ob_shash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - int sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*int sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); '''), @@ -349,7 +346,7 @@ char * data; int len; PyType_Ready(&PyStringArrType_Type); - + data = PyString_AS_STRING(args); len = PyString_GET_SIZE(args); if (data == NULL || len < 1) @@ -373,7 +370,6 @@ const char *dptr, *ip; int len; PyObject *new; - PyObject *ret; ip = dptr = PyString_AS_STRING(self); len = PyString_GET_SIZE(self); @@ -394,7 +390,6 @@ const char *dptr, *ip; int len; PyObject *new; - PyObject *ret; ip = dptr = PyString_AS_STRING(self); len = PyString_GET_SIZE(self); @@ -415,7 +410,6 @@ PyTypeObject *type = &PyStringArrType_Type; PyObject *obj; void *destptr; - int type_num; int itemsize = n; obj = type->tp_alloc(type, itemsize); if (obj == NULL) { diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -7,8 +7,6 @@ from pypy import pypydir from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator import platform from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir from pypy.module.cpyext import api @@ -18,20 +16,7 @@ from rpython.tool import leakfinder from rpython.rlib import rawrefcount -def setup_module(module): - if os.name == 'nt': - # Do not open dreaded dialog box on segfault - import ctypes - SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN - old_err_mode = ctypes.windll.kernel32.GetErrorMode() - new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX - ctypes.windll.kernel32.SetErrorMode(new_err_mode) - module.old_err_mode = old_err_mode - -def teardown_module(module): - if os.name == 'nt': - import ctypes - ctypes.windll.kernel32.SetErrorMode(module.old_err_mode) +from .support import c_compile @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -46,7 +31,30 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def compile_extension_module(space, modname, include_dirs=[], **kwds): +def convert_sources_to_files(sources, dirname): + files = [] + for i, source in enumerate(sources): + filename = dirname / ('source_%d.c' % i) + with filename.open('w') as f: + f.write(str(source)) + files.append(filename) + return files + +def create_so(modname, include_dirs, source_strings=None, source_files=None, + compile_extra=None, link_extra=None, libraries=None): + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=compile_extra, link_extra=link_extra, + include_dirs=include_dirs, + libraries=libraries) + return soname + +def compile_extension_module(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -60,38 +68,36 @@ state = space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': - kwds["libraries"] = [api_library] + libraries = [api_library] # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] + compile_extra = ["/we4013"] # prevent linking with PythonXX.lib w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] - kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % (space.int_w(w_maj), space.int_w(w_min))] - elif sys.platform == 'darwin': - kwds["link_files"] = [str(api_library + '.dylib')] else: - kwds["link_files"] = [str(api_library + '.so')] + libraries = [] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror", "-g", "-O0"] - kwds["link_extra"]=["-g"] + compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + link_extra = ["-g"] + else: + compile_extra = link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs=api.include_dirs + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=api.include_dirs + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra, + libraries=libraries) from pypy.module.imp.importing import get_so_extension pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) soname.rename(pydname) return str(pydname) -def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds): +def compile_extension_module_applevel(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -103,24 +109,23 @@ build the module (so specify your source with one of those). """ if sys.platform == 'win32': - kwds["compile_extra"] = ["/we4013"] - kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] + compile_extra = ["/we4013"] + link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] elif sys.platform == 'darwin': + compile_extra = link_extra = None pass elif sys.platform.startswith('linux'): - kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"] + compile_extra = [ + "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] + link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs = [space.include_dir] + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=[space.include_dir] + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra) return str(soname) def freeze_refcnts(self): @@ -285,8 +290,8 @@ separate_module_sources = [] pydname = self.compile_extension_module( space, name, - separate_module_files=separate_module_files, - separate_module_sources=separate_module_sources) + source_files=separate_module_files, + source_strings=separate_module_sources) return space.wrap(pydname) @gateway.unwrap_spec(name=str, init='str_or_None', body=str, @@ -315,6 +320,11 @@ /* fix for cpython 2.7 Python.h if running tests with -A since pypy compiles with -fvisibility-hidden */ #undef PyMODINIT_FUNC + #ifdef __GNUC__ + # define RPY_EXPORTED extern __attribute__((visibility("default"))) + #else + # define RPY_EXPORTED extern __declspec(dllexport) + #endif #define PyMODINIT_FUNC RPY_EXPORTED void %(body)s @@ -326,16 +336,16 @@ """ % dict(name=name, init=init, body=body, PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' if PY_SSIZE_T_CLEAN else '') - kwds = dict(separate_module_sources=[code]) + kwds = dict(source_strings=[code]) else: assert not PY_SSIZE_T_CLEAN if filename is None: filename = name filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(separate_module_files=[filename]) - kwds['include_dirs'] = include_dirs - mod = self.compile_extension_module(space, name, **kwds) + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module(space, name, + include_dirs=include_dirs, **kwds) if load_it: if self.runappdirect: @@ -975,7 +985,7 @@ ('bar', 'METH_NOARGS', ''' /* reuse a name that is #defined in structmember.h */ - int RO; + int RO = 0; (void)RO; Py_RETURN_NONE; ''' ), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -142,7 +142,7 @@ 2000, 6, 6, 6, 6, 6, 6, Py_None, PyDateTimeAPI->DateTimeType); """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.new_date() == datetime.date(2000, 6, 6) assert module.new_time() == datetime.time(6, 6, 6, 6) @@ -241,6 +241,9 @@ PyObject* obj = PyDelta_FromDSU(6, 6, 6); PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 + // These macros are only defined in CPython 3.x and PyPy. + // See: http://bugs.python.org/issue13727 PyDateTime_DELTA_GET_DAYS(obj); PyDateTime_DELTA_GET_DAYS(delta); @@ -249,10 +252,10 @@ PyDateTime_DELTA_GET_MICROSECONDS(obj); PyDateTime_DELTA_GET_MICROSECONDS(delta); - +#endif return obj; """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.test_date_macros() == datetime.date(2000, 6, 6) assert module.test_datetime_macros() == datetime.datetime( diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py --- a/pypy/module/cpyext/test/test_frameobject.py +++ b/pypy/module/cpyext/test/test_frameobject.py @@ -13,7 +13,7 @@ PyObject *empty_string = PyString_FromString(""); PyObject *empty_tuple = PyTuple_New(0); PyCodeObject *py_code; - PyFrameObject *py_frame; + PyFrameObject *py_frame = NULL; py_code = PyCode_New( 0, /*int argcount,*/ @@ -75,7 +75,7 @@ """ int check; PyObject *type, *value, *tb; - PyObject *ret = PyRun_String("XXX", Py_eval_input, + PyObject *ret = PyRun_String("XXX", Py_eval_input, Py_None, Py_None); if (ret) { Py_DECREF(ret); diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -149,7 +149,6 @@ pybuffer = self.import_parser( ''' Py_buffer buf1, buf2, buf3; - PyObject *result; if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) { return NULL; } diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.micronumpy.ndarray import W_NDimArray from pypy.module.micronumpy.descriptor import get_dtype_cache -import pypy.module.micronumpy.constants as NPY +import pypy.module.micronumpy.constants as NPY def scalar(space): dtype = get_dtype_cache(space).w_float64dtype @@ -237,7 +237,7 @@ except: skip('numpy not importable') else: - numpy_incl = os.path.abspath(os.path.dirname(__file__) + + numpy_incl = os.path.abspath(os.path.dirname(__file__) + '/../include/_numpypy') assert os.path.exists(numpy_incl) cls.w_numpy_include = cls.space.wrap([numpy_incl]) @@ -273,7 +273,7 @@ { /* Should have failed */ Py_DECREF(obj1); - return NULL; + return NULL; } return obj1; ''' @@ -300,14 +300,14 @@ ), ("test_DescrFromType", "METH_O", """ - Signed typenum = PyInt_AsLong(args); + long typenum = PyInt_AsLong(args); return PyArray_DescrFromType(typenum); """ ), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -315,7 +315,7 @@ #define PyArray_FromObject _PyArray_FromObject #define PyArray_FromAny _PyArray_FromAny #endif - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -349,14 +349,14 @@ Py_INCREF(obj); return obj; '''), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -403,14 +403,14 @@ void *array_data[] = {NULL, NULL}; return PyUFunc_FromFuncAndDataAndSignature(funcs, array_data, types, 1, 1, 1, PyUFunc_None, - "float_3x3", - "a ufunc that tests a more complicated signature", + "float_3x3", + "a ufunc that tests a more complicated signature", 0, "(m,m)->(m,m)"); """), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -480,7 +480,7 @@ res += +10; *((float *)args[1]) = res; }; - + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -127,12 +127,12 @@ test_compare(1, 2) test_compare(2, 2) test_compare('2', '1') - + w_i = space.wrap(1) assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() - + def test_IsInstance(self, space, api): assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 @@ -165,7 +165,7 @@ return File""") w_f = space.call_function(w_File) assert api.PyObject_AsFileDescriptor(w_f) == 42 - + def test_hash(self, space, api): assert api.PyObject_Hash(space.wrap(72)) == 72 assert api.PyObject_Hash(space.wrap(-1)) == -1 @@ -250,7 +250,7 @@ if (copy != orig) PyObject_Free(copy); PyObject_Free(orig); - return ret; + return ret; """)]) x = module.realloctest() assert x == 'hello world\x00' @@ -425,7 +425,6 @@ """ Py_buffer buf; PyObject *str = PyString_FromString("hello, world."); - PyObject *result; if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13, 1, PyBUF_WRITABLE)) { diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -130,7 +130,7 @@ module = self.import_extension('foo', [ ("run", "METH_NOARGS", """ - long prev, next; + long prev; PyObject *t = PyTuple_New(1); prev = Py_True->ob_refcnt; Py_INCREF(Py_True); diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,8 +5,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr -import sys - class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): import sys @@ -737,7 +735,6 @@ """ IntLikeObject *intObj; int intval; - PyObject *name; if (!PyArg_ParseTuple(args, "i", &intval)) return NULL; @@ -897,7 +894,7 @@ module.footype("X", (object,), {}) def test_app_subclass_of_c_type(self): - # on cpython, the size changes (6 bytes added) + import sys module = self.import_module(name='foo') size = module.size_of_instances(module.fooType) class f1(object): @@ -907,7 +904,11 @@ class bar(f1, f2): pass assert bar.__base__ is f2 - assert module.size_of_instances(bar) == size + # On cpython, the size changes. + if '__pypy__' in sys.builtin_module_names: + assert module.size_of_instances(bar) == size + else: + assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): module = self.import_module(name='foo') @@ -1058,7 +1059,6 @@ module = self.import_extension('foo', [ ("getMetaClass", "METH_NOARGS", ''' - PyObject *obj; FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT; FooType_Type.tp_base = &PyType_Type; if (PyType_Ready(&FooType_Type) < 0) return NULL; diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -35,7 +35,7 @@ ("test_GetSize_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyUnicode_GetSize(f); + PyUnicode_GetSize(f); Py_DECREF(f); return NULL; @@ -57,7 +57,6 @@ """ PyObject *s, *t; Py_UNICODE* c; - Py_ssize_t len; s = PyUnicode_FromUnicode(NULL, 4); if (s == NULL) @@ -84,7 +83,7 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyUnicodeObject*)obj)->hash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ]) @@ -234,13 +233,13 @@ w_res = api.PyUnicode_AsUTF8String(w_u) assert space.type(w_res) is space.w_str assert space.unwrap(w_res) == 'sp\tm' - + def test_decode_utf8(self, space, api): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) w_u = api.PyUnicode_DecodeUTF8(u, 5, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == u'sp\x134m' - + w_u = api.PyUnicode_DecodeUTF8(u, 2, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == 'sp' @@ -405,7 +404,7 @@ ustr = "abcdef" w_ustr = space.wrap(ustr.decode("ascii")) result = api.PyUnicode_AsASCIIString(w_ustr) - + assert space.eq_w(space.wrap(ustr), result) w_ustr = space.wrap(u"abcd\xe9f") diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -478,7 +478,7 @@ self.intbound = info def _generate_guards_unkown(self, other, box, runtime_box, extra_guards, - optimizer): + state): other_intbound = None if isinstance(other, NotVirtualStateInfoInt): other_intbound = other.intbound @@ -490,7 +490,7 @@ self.intbound.contains(runtime_box.getint())): # this may generate a few more guards than needed, but they are # optimized away when emitting them - self.intbound.make_guards(box, extra_guards, optimizer) + self.intbound.make_guards(box, extra_guards, state.optimizer) return raise VirtualStatesCantMatch("intbounds don't match") diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 11:11:26 2016 From: pypy.commits at gmail.com (rlamy) Date: Thu, 23 Jun 2016 08:11:26 -0700 (PDT) Subject: [pypy-commit] pypy py3k: fixes Message-ID: <576bfc1e.06a81c0a.4bea2.771d@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85353:bb7d9f0e7453 Date: 2016-06-23 15:41 +0100 http://bitbucket.org/pypy/pypy/changeset/bb7d9f0e7453/ Log: fixes diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -80,7 +80,7 @@ if (t == NULL) return NULL; Py_DECREF(t); - c = PyBytes_AsString(s); + c = PyBytes_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -75,7 +75,6 @@ w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type) w_obj.__init__(s) - py_uni.c_hash = space.hash_w(w_obj) track_reference(space, py_obj, w_obj) return w_obj From pypy.commits at gmail.com Thu Jun 23 11:14:26 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:26 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: call base methods consistently in make_guards Message-ID: <576bfcd2.9a4a1c0a.c8829.ffff80e2@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85354:6846f0cd8655 Date: 2016-06-22 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/6846f0cd8655/ Log: call base methods consistently in make_guards diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -117,6 +117,7 @@ def make_guards(self, op, short, optimizer): op = ResOperation(rop.GUARD_NONNULL, [op]) short.append(op) + return PtrInfo.make_guards(self, op, short, optimizer) class AbstractVirtualPtrInfo(NonNullPtrInfo): _attrs_ = ('_cached_vinfo', 'descr', '_is_virtual') @@ -327,20 +328,17 @@ return visitor.visit_virtual(self.descr, fielddescrs) def make_guards(self, op, short, optimizer): + AbstractStructPtrInfo.make_guards(self, op, short, optimizer) if self._known_class is not None: - short.append(ResOperation(rop.GUARD_NONNULL, [op])) if not optimizer.cpu.remove_gctypeptr: short.append(ResOperation(rop.GUARD_IS_OBJECT, [op])) short.append(ResOperation(rop.GUARD_CLASS, [op, self._known_class])) elif self.descr is not None: - short.append(ResOperation(rop.GUARD_NONNULL, [op])) if not optimizer.cpu.remove_gctypeptr: short.append(ResOperation(rop.GUARD_IS_OBJECT, [op])) short.append(ResOperation(rop.GUARD_SUBCLASS, [op, ConstInt(self.descr.get_vtable())])) - else: - AbstractStructPtrInfo.make_guards(self, op, short, optimizer) class StructPtrInfo(AbstractStructPtrInfo): def __init__(self, descr, is_virtual=False): @@ -348,12 +346,10 @@ self._is_virtual = is_virtual def make_guards(self, op, short, optimizer): + AbstractStructPtrInfo.make_guards(self, op, short, optimizer) if self.descr is not None: c_typeid = ConstInt(self.descr.get_type_id()) - short.extend([ - ResOperation(rop.GUARD_NONNULL, [op]), - ResOperation(rop.GUARD_GC_TYPE, [op, c_typeid]) - ]) + short.append(ResOperation(rop.GUARD_GC_TYPE, [op, c_typeid])) @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): @@ -675,6 +671,8 @@ return self._const def make_guards(self, op, short, optimizer): + # don't call base method, all other guards are irrelevant if it's a + # constant short.append(ResOperation(rop.GUARD_VALUE, [op, self._const])) def _get_info(self, descr, optheap): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -454,6 +454,17 @@ """ self.compare(guards, expected, [box]) + def test_guard_nonnull(self): + value1 = info.NonNullPtrInfo() + box = InputArgRef() + guards = [] + value1.make_guards(box, guards, FakeOptimizer(self.cpu)) + expected = """ + [p0] + guard_nonnull(p0) [] + """ + self.compare(guards, expected, [box]) + def test_equal_inputargs(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) From pypy.commits at gmail.com Thu Jun 23 11:14:28 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:28 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: start emitting guards from infos, bit of a pain Message-ID: <576bfcd4.9a4a1c0a.c8829.ffff80e8@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85355:8729edc0c66a Date: 2016-06-22 14:56 +0200 http://bitbucket.org/pypy/pypy/changeset/8729edc0c66a/ Log: start emitting guards from infos, bit of a pain diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -24,6 +24,7 @@ return None raise AssertionError("bad rettype") + class CompatibilityCondition(object): """ A collections of conditions that an object needs to fulfil. """ def __init__(self, ptr): @@ -105,10 +106,17 @@ return copied_op, QuasiimmutGetfieldAndPureCallCondition( op, qmutdescr, optimizer) + def emit_conditions(self, op, short, optimizer): + """ re-emit the conditions about variable op into the short preamble + """ + for cond in self.conditions: + cond.emit_condition(op, short, optimizer) + def repr_of_conditions(self, argrepr="?"): return "\n".join([cond.repr(argrepr) for cond in self.conditions]) + class Condition(object): def __init__(self, optimizer): self.metainterp_sd = optimizer.metainterp_sd @@ -138,6 +146,9 @@ def repr(self): return "" + def emit_condition(self, op, short, optimizer): + raise NotImplementedError("abstract base class") + def _repr_const(self, arg): from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr from rpython.rtyper.annlowlevel import llstr, hlstr @@ -211,6 +222,30 @@ return False return True + def emit_condition(self, op, short, optimizer): + from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID + from rpython.jit.metainterp.resoperation import rop, ResOperation + # woah, mess + args = self.args[:] + args[1] = op + descr = self.descr + rettype = descr.get_result_type() + if rettype == INT: + call_op = ResOperation(rop.CALL_PURE_I, args, descr) + elif rettype == FLOAT: + call_op = ResOperation(rop.CALL_PURE_F, args, descr) + elif rettype == REF: + call_op = ResOperation(rop.CALL_PURE_R, args, descr) + else: + assert rettype == VOID + # XXX maybe we should forbid this + call_op = ResOperation(rop.CALL_PURE_R, args, descr) + short.append(call_op) + return + short.append(call_op) + short.append(ResOperation(rop.GUARD_VALUE, [call_op, self.res])) + + def repr(self, argrepr="?"): addr = self.args[0].getaddr() funcname = self.metainterp_sd.get_name_from_address(addr) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -78,7 +78,13 @@ pass def make_guards(self, op, short, optimizer): - pass + compat_cond = self._compatibility_conditions + if compat_cond is None: + return + short.append( + ResOperation(rop.GUARD_COMPATIBLE, [ + op, compat_cond.known_valid])) + compat_cond.emit_conditions(op, short, optimizer) @specialize.arg(2) def get_constant_string_spec(self, string_optimizer, mode): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -26,6 +26,8 @@ def __init__(self, cpu): self.cpu = cpu self.optearlyforce = None + self.metainterp_sd = None + self._last_debug_merge_point = None class BaseTestGenerateGuards(BaseTest): def setup_class(self): @@ -465,6 +467,45 @@ """ self.compare(guards, expected, [box]) + def test_guard_compatible(self): + from rpython.jit.metainterp.compatible import CompatibilityCondition + value1 = info.PtrInfo() + ptr = "fakeptr" + value1._compatibility_conditions = CompatibilityCondition( + ConstPtr(self.myptr)) + box = InputArgRef() + guards = [] + value1.make_guards(box, guards, FakeOptimizer(self.cpu)) + expected = """ + [p0] + guard_compatible(p0, ConstPtr(myptr)) [] + """ + self.compare(guards, expected, [box]) + + def test_guard_compatible_with_conditions(self): + from rpython.jit.metainterp.compatible import CompatibilityCondition + optimizer = FakeOptimizer(self.cpu) + value1 = info.PtrInfo() + ptr = "fakeptr" + ccond = value1._compatibility_conditions = CompatibilityCondition( + ConstPtr(self.myptr)) + op = ResOperation( + rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.myptr)], + descr=self.plaincalldescr) + copied_op, cond = ccond.prepare_const_arg_call( + op, optimizer) + ccond.record_condition(cond, ConstInt(5), optimizer) + box = InputArgRef() + guards = [] + value1.make_guards(box, guards, FakeOptimizer(self.cpu)) + expected = """ + [p0] + guard_compatible(p0, ConstPtr(myptr)) [] + i1 = call_pure_i(123, p0, descr=plaincalldescr) + guard_value(i1, 5) [] + """ + self.compare(guards, expected, [box]) + def test_equal_inputargs(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) From pypy.commits at gmail.com Thu Jun 23 11:14:30 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:30 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: produce guards for QuasiimmutGetfieldAndPureCallCondition Message-ID: <576bfcd6.05b01c0a.142c4.ffff8e4c@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85356:3c07b3cda170 Date: 2016-06-22 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/3c07b3cda170/ Log: produce guards for QuasiimmutGetfieldAndPureCallCondition (more mess) diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -270,6 +270,8 @@ def __init__(self, op, qmutdescr, optimizer): PureCallCondition.__init__(self, op, optimizer) self.args[2] = None + # XXX not 100% sure whether it's save to store the whole descr + self.qmutdescr = qmutdescr self.qmut = qmutdescr.qmut self.mutatefielddescr = qmutdescr.mutatefielddescr self.fielddescr = qmutdescr.fielddescr @@ -330,6 +332,33 @@ return False return True + def emit_condition(self, op, short, optimizer): + from rpython.jit.metainterp.resoperation import rop, ResOperation + # more mess + fielddescr = self.fielddescr + if fielddescr.is_pointer_field(): + getfield_op = ResOperation( + rop.GETFIELD_GC_R, [op], fielddescr) + elif fielddescr.is_float_field(): + getfield_op = ResOperation( + rop.GETFIELD_GC_F, [op], fielddescr) + else: + getfield_op = ResOperation( + rop.GETFIELD_GC_I, [op], fielddescr) + short.extend([ + ResOperation( + rop.QUASIIMMUT_FIELD, [op], self.qmutdescr), + ResOperation( + rop.GUARD_NOT_INVALIDATED, []), + getfield_op]) + index = len(short) + PureCallCondition.emit_condition(self, op, short, optimizer) + call_op = short[index] + assert call_op.opnum in ( + rop.CALL_PURE_I, rop.CALL_PURE_R, + rop.CALL_PURE_F, rop.CALL_PURE_N) + call_op.setarg(2, getfield_op) + def repr(self, argrepr="?"): addr = self.args[0].getaddr() funcname = self.metainterp_sd.get_name_from_address(addr) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -28,6 +28,7 @@ self.optearlyforce = None self.metainterp_sd = None self._last_debug_merge_point = None + self.quasi_immutable_deps = None class BaseTestGenerateGuards(BaseTest): def setup_class(self): @@ -488,21 +489,43 @@ value1 = info.PtrInfo() ptr = "fakeptr" ccond = value1._compatibility_conditions = CompatibilityCondition( - ConstPtr(self.myptr)) + ConstPtr(self.quasiptr)) + + # regular call op = ResOperation( - rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.myptr)], + rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr)], descr=self.plaincalldescr) copied_op, cond = ccond.prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) + + # call with quasi-immut + box = InputArgRef() + ccond.register_quasi_immut_field( + ResOperation(rop.QUASIIMMUT_FIELD, [box], self.quasiimmutdescr)) + getfield_op = ResOperation( + rop.GETFIELD_GC_I, [box], self.quasifielddescr) + op = ResOperation( + rop.CALL_PURE_I, + [ConstInt(123), ConstPtr(self.quasiptr), getfield_op], + descr=self.nonwritedescr) + copied_op, cond = ccond.prepare_const_arg_call( + op, optimizer) + ccond.record_condition(cond, ConstInt(5), optimizer) + box = InputArgRef() guards = [] value1.make_guards(box, guards, FakeOptimizer(self.cpu)) expected = """ [p0] - guard_compatible(p0, ConstPtr(myptr)) [] + guard_compatible(p0, ConstPtr(quasiptr)) [] i1 = call_pure_i(123, p0, descr=plaincalldescr) guard_value(i1, 5) [] + quasiimmut_field(p0, descr=quasiimmutdescr) + guard_not_invalidated() [] + i0 = getfield_gc_i(p0, descr=quasifielddescr) + i2 = call_pure_i(123, p0, i0, descr=nonwritedescr) + guard_value(i2, 5) [] """ self.compare(guards, expected, [box]) From pypy.commits at gmail.com Thu Jun 23 11:14:31 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:31 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: start doing loop-invariant code motion of guard_compatible Message-ID: <576bfcd7.0c091c0a.64506.794b@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85357:625f490451ee Date: 2016-06-22 17:14 +0200 http://bitbucket.org/pypy/pypy/changeset/625f490451ee/ Log: start doing loop-invariant code motion of guard_compatible (something is currently very broken, but wanted to commit an intermediate step) diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -33,15 +33,26 @@ self.last_quasi_immut_field_op = None # -1 means "stay on the original trace" self.jump_target = -1 + self.frozen = False + + def frozen_copy(self): + res = CompatibilityCondition(self.known_valid) + res.conditions = self.conditions[:] + assert self.jump_target == -1 + res.frozen = True + return res def record_condition(self, cond, res, optimizer): for oldcond in self.conditions: if oldcond.same_cond(cond, res): - return + return True + if self.frozen: + False cond.activate(res, optimizer) if self.conditions and self.conditions[-1].debug_mp_str == cond.debug_mp_str: cond.debug_mp_str = '' self.conditions.append(cond) + return True def register_quasi_immut_field(self, op): self.last_quasi_immut_field_op = op diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -121,8 +121,8 @@ assert self.get_last_guard(optimizer).is_guard() def make_guards(self, op, short, optimizer): - op = ResOperation(rop.GUARD_NONNULL, [op]) - short.append(op) + guard_op = ResOperation(rop.GUARD_NONNULL, [op]) + short.append(guard_op) return PtrInfo.make_guards(self, op, short, optimizer) class AbstractVirtualPtrInfo(NonNullPtrInfo): diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -147,10 +147,12 @@ if copied_op: result = self._can_optimize_call_pure(copied_op) if result is not None: - self.make_constant(op, result) - self.last_emitted_operation = REMOVED - ccond.record_condition(cond, result, self.optimizer) - return + recorded = ccond.record_condition( + cond, result, self.optimizer) + if recorded: + self.make_constant(op, result) + self.last_emitted_operation = REMOVED + return # Step 1: check if all arguments are constant for arg in op.getarglist(): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py --- a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py @@ -302,3 +302,80 @@ """ self.optimize_loop(ops, expected, expected_preamble=preamble) + + def test_guard_compatible_call_pure(self): + call_pure_results = { + (ConstInt(123), ConstPtr(self.myptr)): ConstInt(5), + (ConstInt(124), ConstPtr(self.myptr)): ConstInt(7), + } + ops = """ + [p1] + guard_compatible(p1, ConstPtr(myptr)) [] + i3 = call_pure_i(123, p1, descr=plaincalldescr) + escape_n(i3) + guard_compatible(p1, ConstPtr(myptr)) [] + i5 = call_pure_i(124, p1, descr=plaincalldescr) + escape_n(i5) + jump(p1) + """ + preamble = """ + [p1] + guard_compatible(p1, ConstPtr(myptr)) [] + escape_n(5) + escape_n(7) + jump(p1) + """ + expected = """ + [p0] + escape_n(5) + escape_n(7) + jump(p0) + """ + self.optimize_loop(ops, expected, expected_preamble=preamble, call_pure_results=call_pure_results) + # whitebox-test the guard_compatible descr a bit + descr = self.preamble.operations[1].getdescr() + assert descr._compatibility_conditions is not None + assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.myptr)) + assert len(descr._compatibility_conditions.conditions) == 2 + + def test_quasiimmut(self): + ops = """ + [p1] + guard_compatible(p1, ConstPtr(quasiptr)) [] + quasiimmut_field(p1, descr=quasiimmutdescr) + guard_not_invalidated() [] + i0 = getfield_gc_i(p1, descr=quasifielddescr) + i1 = call_pure_i(123, p1, i0, descr=nonwritedescr) + quasiimmut_field(p1, descr=quasiimmutdescr) + guard_not_invalidated() [] + i3 = getfield_gc_i(p1, descr=quasifielddescr) + i4 = call_pure_i(123, p1, i3, descr=nonwritedescr) + escape_n(i1) + escape_n(i4) + jump(p1) + """ + preamble = """ + [p1] + guard_compatible(p1, ConstPtr(quasiptr)) [] + guard_not_invalidated() [] + i0 = getfield_gc_i(p1, descr=quasifielddescr) # will be removed by the backend + escape_n(5) + escape_n(5) + jump(p1) + """ + expected = """ + [p1] + guard_not_invalidated() [] + escape_n(5) + escape_n(5) + jump(p1) + """ + + call_pure_results = { + (ConstInt(123), ConstPtr(self.quasiptr), ConstInt(-4247)): ConstInt(5), + } + self.optimize_loop(ops, expected, expected_preamble=preamble, call_pure_results=call_pure_results) + descr = self.preamble.operations[1].getdescr() + assert descr._compatibility_conditions is not None + assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.quasiptr)) + assert len(descr._compatibility_conditions.conditions) == 1 diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -60,6 +60,7 @@ info = self.unroll_and_optimize(loop, call_pure_results, jump_values) preamble = info.preamble + self.preamble = preamble preamble.check_consistency(check_descr=False) # diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -88,7 +88,9 @@ if preamble_info._compatibility_conditions: info_in_loop = op.get_forwarded() if info_in_loop is not None: - info_in_loop._compatibility_conditions = preamble_info._compatibility_conditions + ccond = preamble_info._compatibility_conditions + ccond = ccond.frozen_copy() + info_in_loop._compatibility_conditions = ccond elif isinstance(preamble_info, intutils.IntBound): if preamble_info.lower > MININT/2 or preamble_info.upper < MAXINT/2: intbound = self.getintbound(op) From pypy.commits at gmail.com Thu Jun 23 11:14:33 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:33 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: intermediate checkin: start sketching guard_compatible support in virtualstate Message-ID: <576bfcd9.aa29c20a.f859c.3b0c@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85358:9f336d3981bb Date: 2016-06-23 11:41 +0200 http://bitbucket.org/pypy/pypy/changeset/9f336d3981bb/ Log: intermediate checkin: start sketching guard_compatible support in virtualstate diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -42,12 +42,17 @@ res.frozen = True return res - def record_condition(self, cond, res, optimizer): + def contains_condition(self, cond, res=None): for oldcond in self.conditions: if oldcond.same_cond(cond, res): return True + return False + + def record_condition(self, cond, res, optimizer): + if self.contains_condition(cond, res): + return True if self.frozen: - False + return False cond.activate(res, optimizer) if self.conditions and self.conditions[-1].debug_mp_str == cond.debug_mp_str: cond.debug_mp_str = '' @@ -57,11 +62,17 @@ def register_quasi_immut_field(self, op): self.last_quasi_immut_field_op = op - def check_compat_and_activate(self, cpu, ref, loop_token): + def check_compat(self, cpu, ref): for cond in self.conditions: if not cond.check(cpu, ref): return False - # need to tell all conditions, in case a quasi-immut needs to be registered + return True + + def check_compat_and_activate(self, cpu, ref, loop_token): + if not self.check_compat(cpu, ref): + return False + # need to tell all conditions, in case a quasi-immut needs to be + # registered for cond in self.conditions: cond.activate_secondary(ref, loop_token) return True @@ -123,6 +134,23 @@ for cond in self.conditions: cond.emit_condition(op, short, optimizer) + def emit_needed_conditions_if_const_matches( + self, other, const, op, extra_guards, optimizer, cpu): + """ go through self.conditions. if the condition is present in other, + do nothing. If it is not, check whether ref matches the condition. If + not return False, otherwise emit guards for the condition. Return True + at the end. """ + for cond in self.conditions: + if other is None or not other.contains_condition(cond): + if const is None: + return False + ref = const.getref_base() + if cond.check(cpu, ref): + cond.emit_condition(op, short, optimizer) + else: + return False + return True + def repr_of_conditions(self, argrepr="?"): return "\n".join([cond.repr(argrepr) for cond in self.conditions]) @@ -151,7 +179,7 @@ def activate_secondary(self, ref, loop_token): pass - def same_cond(self, other, res): + def same_cond(self, other, res=None): return False def repr(self): @@ -216,11 +244,13 @@ return False return True - def same_cond(self, other, res): + def same_cond(self, other, res=None): if type(other) is not PureCallCondition: return False if len(self.args) != len(other.args): return False + if res is None: + res = other.res if not self.res.same_constant(res): return False if self.descr is not other.descr: @@ -321,11 +351,13 @@ return False return True - def same_cond(self, other, res): + def same_cond(self, other, res=None): if type(other) is not QuasiimmutGetfieldAndPureCallCondition: return False if len(self.args) != len(other.args): return False + if res is None: + res = other.res if not self.res.same_constant(res): return False if self.descr is not other.descr: diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -355,6 +355,8 @@ if type == 'i': return NotVirtualStateInfoInt(cpu, type, info) if type == 'r': + if info._compatibility_conditions is not None: + return NotVirtualStateInfoPtrCompatible(cpu, type, info) return NotVirtualStateInfoPtr(cpu, type, info) return NotVirtualStateInfo(cpu, type, info) @@ -597,6 +599,29 @@ else: raise VirtualStatesCantMatch("classes don't match") +class NotVirtualStateInfoPtrCompatible(NotVirtualStateInfoPtr): + def __init__(self, cpu, type, info): + ccond = info._compatibility_conditions + self._compatibility_conditions = ccond.frozen_copy() + NotVirtualStateInfoPtr.__init__(self, cpu, type, info) + + def _generate_guards(self, other, box, runtime_box, state): + NotVirtualStateInfoPtr._generate_guards( + self, other, box, runtime_box, state) + ccond_self = self._compatibility_conditions + if isinstance(other, NotVirtualStateInfoPtrCompatible): + ccond_other = other._compatibility_conditions + else: + ccond_other = None + const = None + if runtime_box is not None: + const = runtime_box.constbox() + if ccond_self.emit_needed_conditions_if_const_matches( + ccond_other, const, box, state.extra_guards, + state.optimizer, state.cpu): + return + raise VirtualStatesCantMatch("target compatibility conditions aren't met") + class VirtualState(object): def __init__(self, state): diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -42,8 +42,8 @@ x = self.meta_interp(main, []) assert x < 30 - # trace, two bridges, a finish bridge - self.check_trace_count(4) + # trace, one bridge, a finish bridge + self.check_trace_count(3) def test_simple_check_values(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -87,8 +87,8 @@ x = self.meta_interp(main, []) assert x == main() - # trace, two bridges, a finish bridge - self.check_trace_count(4) + # trace, a bridge, a finish bridge + self.check_trace_count(3) def test_exception(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -122,7 +122,7 @@ f(100, p3) self.meta_interp(main, []) - # XXX check number of bridges + self.check_trace_count(3) def test_quasi_immutable(self): @@ -197,7 +197,7 @@ x = self.meta_interp(main, [False]) assert x < 30 - # XXX check number of bridges + self.check_trace_count(4) def test_dont_record_repeated_guard_compatible(self): @@ -264,8 +264,8 @@ x = self.meta_interp(main, []) assert x < 30 - # trace, two bridges, a finish bridge - self.check_trace_count(4) + # trace, one bridge, a finish bridge + self.check_trace_count(3) def test_order_of_chained_guards(self): From pypy.commits at gmail.com Thu Jun 23 11:14:35 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 23 Jun 2016 08:14:35 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: a test for a bridge that jumps to the loop. causes all kinds of problems. Message-ID: <576bfcdb.094ac20a.6c701.3b9f@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85359:497da7b013d3 Date: 2016-06-23 14:49 +0200 http://bitbucket.org/pypy/pypy/changeset/497da7b013d3/ Log: a test for a bridge that jumps to the loop. causes all kinds of problems. diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -131,6 +131,10 @@ def emit_conditions(self, op, short, optimizer): """ re-emit the conditions about variable op into the short preamble """ + from rpython.jit.metainterp.resoperation import rop, ResOperation + short.append( + ResOperation(rop.GUARD_COMPATIBLE, [ + op, self.known_valid])) for cond in self.conditions: cond.emit_condition(op, short, optimizer) @@ -140,13 +144,22 @@ do nothing. If it is not, check whether ref matches the condition. If not return False, otherwise emit guards for the condition. Return True at the end. """ + from rpython.jit.metainterp.resoperation import rop, ResOperation + have_guard = False for cond in self.conditions: if other is None or not other.contains_condition(cond): if const is None: return False ref = const.getref_base() if cond.check(cpu, ref): - cond.emit_condition(op, short, optimizer) + if not have_guard: + # NB: the guard_compatible here needs to use const, + # otherwise the optimizer will just complain + extra_guards.append(ResOperation( + rop.GUARD_COMPATIBLE, + [op, const])) + have_guard = True + cond.emit_condition(op, extra_guards, optimizer, const) else: return False return True @@ -185,7 +198,7 @@ def repr(self): return "" - def emit_condition(self, op, short, optimizer): + def emit_condition(self, op, short, optimizer, const=None): raise NotImplementedError("abstract base class") def _repr_const(self, arg): @@ -263,7 +276,7 @@ return False return True - def emit_condition(self, op, short, optimizer): + def emit_condition(self, op, short, optimizer, const=None): from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID from rpython.jit.metainterp.resoperation import rop, ResOperation # woah, mess @@ -283,6 +296,11 @@ call_op = ResOperation(rop.CALL_PURE_R, args, descr) short.append(call_op) return + # add result to call_pure_results + if const is not None: + args = args[:] + args[1] = const + optimizer.call_pure_results[args] = self.res short.append(call_op) short.append(ResOperation(rop.GUARD_VALUE, [call_op, self.res])) @@ -375,7 +393,7 @@ return False return True - def emit_condition(self, op, short, optimizer): + def emit_condition(self, op, short, optimizer, const=None): from rpython.jit.metainterp.resoperation import rop, ResOperation # more mess fielddescr = self.fielddescr @@ -395,12 +413,18 @@ rop.GUARD_NOT_INVALIDATED, []), getfield_op]) index = len(short) - PureCallCondition.emit_condition(self, op, short, optimizer) + PureCallCondition.emit_condition(self, op, short, optimizer, const) call_op = short[index] + # puh, not pretty + args = call_op.getarglist() + if const is not None: + del optimizer.call_pure_results[args] assert call_op.opnum in ( rop.CALL_PURE_I, rop.CALL_PURE_R, rop.CALL_PURE_F, rop.CALL_PURE_N) call_op.setarg(2, getfield_op) + if const is not None: + optimizer.call_pure_results[call_op.getarglist()] = self.res def repr(self, argrepr="?"): addr = self.args[0].getaddr() diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -81,9 +81,6 @@ compat_cond = self._compatibility_conditions if compat_cond is None: return - short.append( - ResOperation(rop.GUARD_COMPATIBLE, [ - op, compat_cond.known_valid])) compat_cond.emit_conditions(op, short, optimizer) @specialize.arg(2) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -310,7 +310,10 @@ for guard in extra_guards.extra_guards: if isinstance(guard, GuardResOp): guard.rd_resume_position = patchguardop.rd_resume_position - guard.setdescr(compile.ResumeAtPositionDescr()) + if guard.opnum == rop.GUARD_COMPATIBLE: + guard.setdescr(compile.GuardCompatibleDescr()) + else: + guard.setdescr(compile.ResumeAtPositionDescr()) self.send_extra_operation(guard) except VirtualStatesCantMatch: continue @@ -375,8 +378,12 @@ sop = short[i] arglist = self._map_args(mapping, sop.getarglist()) if sop.is_guard(): + if sop.opnum == rop.GUARD_COMPATIBLE: + descr = compile.GuardCompatibleDescr() + else: + descr = compile.ResumeAtPositionDescr() op = sop.copy_and_change(sop.getopnum(), arglist, - descr=compile.ResumeAtPositionDescr()) + descr=descr) assert isinstance(op, GuardResOp) op.rd_resume_position = patchguardop.rd_resume_position else: diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -355,7 +355,7 @@ if type == 'i': return NotVirtualStateInfoInt(cpu, type, info) if type == 'r': - if info._compatibility_conditions is not None: + if info is not None and info._compatibility_conditions is not None: return NotVirtualStateInfoPtrCompatible(cpu, type, info) return NotVirtualStateInfoPtr(cpu, type, info) return NotVirtualStateInfo(cpu, type, info) diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -319,5 +319,46 @@ x = self.meta_interp(main, []) # trace, two bridges, a finish bridge + self.check_trace_count(3) + assert x < 30 + + def test_merge(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + p1 = lltype.malloc(S) + p1.x = 1 + + p2 = lltype.malloc(S) + p2.x = 1 + + driver = jit.JitDriver(greens = [], reds = ['n', 'x']) + + class A(object): + pass + + c = A() + c.count = 0 + @jit.elidable_compatible() + def g(s, ignored): + c.count += 1 + return s.x + + def f(n, x): + while n > 0: + driver.can_enter_jit(n=n, x=x) + driver.jit_merge_point(n=n, x=x) + n -= g(x, "abc") + if n & 2: + n -= 2 + + def main(): + g(p1, "def") # make annotator not make argument constant + f(1000, p1) + f(1000, p2) + return c.count + + x = self.meta_interp(main, []) + + assert x < 30 + # trace, two bridges, a finish bridge self.check_trace_count(4) - assert x < 50 + From pypy.commits at gmail.com Thu Jun 23 11:45:09 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 23 Jun 2016 08:45:09 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: (ppc) added impl for float_abs & float_neg. Message-ID: <576c0405.42431c0a.a751f.ffff9468@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85360:0ad6556517cf Date: 2016-06-22 18:10 +0200 http://bitbucket.org/pypy/pypy/changeset/0ad6556517cf/ Log: (ppc) added impl for float_abs & float_neg. diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -62,6 +62,7 @@ XFX = Form("CRM", "rS", "XO1") XLL = Form("LL", "XO1") XX1 = Form("fvrT", "rA", "rB", "XO1") +XX2 = Form("fvrT", "fvrB", "XO5") XX3 = Form("fvrT", "fvrA", "fvrB", "XO9") XV = Form("ivrT", "rA", "rB", "XO1") VX = Form("ivrT", "ivrA", "ivrB", "XO8") @@ -611,6 +612,14 @@ xvdivdp = XX3(60, XO9=102) xvdivsp = XX3(60, XO9=88) + # neg + xvnegdp = XX2(60, XO5=505) + xvabssp = XX2(60, XO5=441) + + # abs + xvabsdp = XX2(60, XO5=473) + xvabssp = XX2(60, XO5=409) + # INTEGER # ------- @@ -644,8 +653,6 @@ # vector move register is alias to vector or vmr = vor - - # shift, perm and select lvsl = XV(31, XO1=6) lvsr = XV(31, XO1=38) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -213,6 +213,26 @@ # TODO self.regalloc_mov(loc0, resloc) + def emit_vec_float_abs(self, op, arglocs, resloc): + resloc, argloc, sizeloc = arglocs + size = sizeloc.value + if size == 4: + self.mc.xvabssp(resloc.value, argloc.value) + elif size == 8: + self.mc.xvabsdp(resloc.value, argloc.value) + else: + notimplemented("[ppc/assembler] float abs for size %d" % size) + + def emit_vec_float_neg(self, op, arglocs, resloc): + resloc, argloc, sizeloc = arglocs + size = sizeloc.value + if size == 4: + self.mc.xvnegsp(resloc.value, argloc.value) + elif size == 8: + self.mc.xvnegdp(resloc.value, argloc.value) + else: + notimplemented("[ppc/assembler] float neg for size %d" % size) + #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc): # self.implement_guard(guard_token) @@ -328,22 +348,6 @@ # # entries before) become ones # self.mc.PCMPEQ(loc, temp, sizeloc.value) - #def genop_vec_float_abs(self, op, arglocs, resloc): - # src, sizeloc = arglocs - # size = sizeloc.value - # if size == 4: - # self.mc.ANDPS(src, heap(self.single_float_const_abs_addr)) - # elif size == 8: - # self.mc.ANDPD(src, heap(self.float_const_abs_addr)) - - #def genop_vec_float_neg(self, op, arglocs, resloc): - # src, sizeloc = arglocs - # size = sizeloc.value - # if size == 4: - # self.mc.XORPS(src, heap(self.single_float_const_neg_addr)) - # elif size == 8: - # self.mc.XORPD(src, heap(self.float_const_neg_addr)) - #def genop_vec_float_eq(self, op, arglocs, resloc): # _, rhsloc, sizeloc = arglocs # size = sizeloc.value @@ -659,17 +663,16 @@ - #def prepare_vec_arith_unary(self, op): - # lhs = op.getarg(0) - # assert isinstance(lhs, VectorOp) - # args = op.getarglist() - # res = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # self.perform(op, [res, imm(lhs.bytesize)], res) + def prepare_vec_arith_unary(self, op): + a0 = op.getarg(0) + loc0 = self.ensure_vector_reg(a0) + resloc = self.force_allocate_vector_reg(op) + sizeloc = imm(op.bytesize) + return [resloc, loc0, sizeloc] - #prepare_vec_float_neg = prepare_vec_arith_unary - #prepare_vec_float_abs = prepare_vec_arith_unary - #del prepare_vec_arith_unary - + prepare_vec_float_neg = prepare_vec_arith_unary + prepare_vec_float_abs = prepare_vec_arith_unary + del prepare_vec_arith_unary #def prepare_vec_float_eq(self, op): # assert isinstance(op, VectorOp) From pypy.commits at gmail.com Thu Jun 23 11:45:11 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 23 Jun 2016 08:45:11 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: catchup default Message-ID: <576c0407.e655c20a.55388.45dc@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85361:3fa201982b38 Date: 2016-06-23 17:44 +0200 http://bitbucket.org/pypy/pypy/changeset/3fa201982b38/ Log: catchup default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,9 @@ Allow rw access to the char* returned from PyString_AS_STRING, also refactor PyStringObject to look like cpython's and allow subclassing PyString_Type and PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Thu Jun 23 12:16:28 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 23 Jun 2016 09:16:28 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: printing rewritten for PYPYLOG again Message-ID: <576c0b5c.c5301c0a.4b84b.ffffa38e@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85362:2816d1b93aad Date: 2016-06-23 18:15 +0200 http://bitbucket.org/pypy/pypy/changeset/2816d1b93aad/ Log: printing rewritten for PYPYLOG again diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -544,6 +544,11 @@ if logger: log = logger.log_trace(MARK_TRACE_ASM, None, self.mc) log.write(inputargs, operations, ops_offset=ops_offset) + + # legacy + if log.logger_ops: + log.logger_ops.log_loop(inputargs, operations, 0, "rewritten", + name=loopname, ops_offset=ops_offset) self.fixup_target_tokens(rawstart) self.teardown() @@ -613,6 +618,11 @@ log.write(inputargs, operations, ops_offset) # log that the already written bridge is stitched to a descr! logger.log_patch_guard(descr_number, rawstart) + + # legacy + if log.logger_ops: + log.logger_ops.log_bridge(inputargs, operations, "rewritten", + faildescr, ops_offset=ops_offset) self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) self.teardown() diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -485,6 +485,9 @@ def do_compile_loop(jd_id, unique_id, metainterp_sd, inputargs, operations, looptoken, log=True, name='', memo=None): + # legacy + metainterp_sd.logger_ops.log_loop(inputargs, operations, -2, + 'compiling', None, name, memo) _log = metainterp_sd.jitlog.log_trace(MARK_TRACE_OPT, metainterp_sd, None) _log.write(inputargs, operations) return metainterp_sd.cpu.compile_loop(inputargs, @@ -495,6 +498,9 @@ def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token, log=True, memo=None): + # legacy + metainterp_sd.logger_ops.log_bridge(inputargs, operations, "compiling", + memo=memo) _log = metainterp_sd.jitlog.log_trace(MARK_TRACE_OPT, metainterp_sd, None) _log.write(inputargs, operations) assert isinstance(faildescr, AbstractFailDescr) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1761,6 +1761,9 @@ self.jitlog = jl.VMProfJitLogger(self.cpu) self.logger_noopt = Logger(self) self.logger_ops = Logger(self, guard_number=True) + # legacy loggers + self.jitlog.logger_noopt = self.logger_noopt + self.jitlog.logger_ops = self.logger_ops self.profiler = ProfilerClass() self.profiler.cpu = cpu diff --git a/rpython/rlib/jitlog.py b/rpython/rlib/jitlog.py --- a/rpython/rlib/jitlog.py +++ b/rpython/rlib/jitlog.py @@ -240,6 +240,9 @@ self.trace_id = -1 self.metainterp_sd = None self.cintf = _get_vmprof().cintf + # legacy + self.logger_ops = None + self.logger_noopt = None def setup_once(self): if self.cintf.jitlog_enabled(): From pypy.commits at gmail.com Thu Jun 23 14:22:29 2016 From: pypy.commits at gmail.com (rlamy) Date: Thu, 23 Jun 2016 11:22:29 -0700 (PDT) Subject: [pypy-commit] pypy py3k: fix bad merge (yet again) Message-ID: <576c28e5.50991c0a.a2fa4.71b2@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85363:425c51ee5fb7 Date: 2016-06-23 19:21 +0100 http://bitbucket.org/pypy/pypy/changeset/425c51ee5fb7/ Log: fix bad merge (yet again) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -79,7 +79,7 @@ called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_bytes.layout.typedef) - py_obj = typedescr.allocate(space, space.w_bytes) + py_obj = typedescr.allocate(space, space.w_bytes, length) py_str = rffi.cast(PyBytesObject, py_obj) py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -32,7 +32,7 @@ result = 1; } #ifdef PYPY_VERSION - expected_size = sizeof(void*)*7; + expected_size = 48; #elif defined Py_DEBUG expected_size = 53; #else From pypy.commits at gmail.com Thu Jun 23 16:44:10 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 23 Jun 2016 13:44:10 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <576c4a1a.6a25c20a.9c02e.ffffada2@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r762:4d414b2f4234 Date: 2016-06-23 22:45 +0200 http://bitbucket.org/pypy/pypy.org/changeset/4d414b2f4234/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $64596 of $105000 (61.5%) + $64615 of $105000 (61.5%)
    @@ -23,7 +23,7 @@
  • diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -17,7 +17,7 @@ 2nd call: - $30744 of $80000 (38.4%) + $30754 of $80000 (38.4%)
    @@ -25,7 +25,7 @@
  • From pypy.commits at gmail.com Fri Jun 24 02:40:09 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 23 Jun 2016 23:40:09 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: fix memory leak, define and use fast path for dealloc Message-ID: <576cd5c9.a758c20a.e6c1a.3c0a@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85364:57294d559e31 Date: 2016-06-23 22:39 +0300 http://bitbucket.org/pypy/pypy/changeset/57294d559e31/ Log: fix memory leak, define and use fast path for dealloc diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -122,8 +122,8 @@ for i in range(py_tup.c_ob_size): if p[i] and p[i].c_ob_refcnt > 0: decref(space, p[i]) - while py_obj.c_ob_refcnt > 0: - decref(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ From pypy.commits at gmail.com Fri Jun 24 02:40:10 2016 From: pypy.commits at gmail.com (mattip) Date: Thu, 23 Jun 2016 23:40:10 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: use faster dealloc in all cases Message-ID: <576cd5ca.4dd11c0a.66010.7240@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85365:e5c07c3edeb3 Date: 2016-06-23 22:39 +0300 http://bitbucket.org/pypy/pypy/changeset/e5c07c3edeb3/ Log: use faster dealloc in all cases diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -26,54 +26,10 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -120,8 +120,8 @@ def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -44,8 +44,8 @@ Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -599,7 +599,7 @@ @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): - from pypy.module.cpyext.object import PyObject_dealloc + from pypy.module.cpyext.object import _dealloc obj_pto = rffi.cast(PyTypeObjectPtr, obj) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) @@ -610,7 +610,7 @@ heaptype = rffi.cast(PyHeapTypeObject, obj) Py_DecRef(space, heaptype.c_ht_name) Py_DecRef(space, base_pyo) - PyObject_dealloc(space, obj) + _dealloc(space, obj) def type_alloc(space, w_metatype, itemsize=0): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -90,8 +90,8 @@ Py_DecRef(space, py_unicode.c_defenc) if py_unicode.c_str: lltype.free(py_unicode.c_str, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): From pypy.commits at gmail.com Fri Jun 24 04:14:33 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 24 Jun 2016 01:14:33 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: called getattr on wrong instance Message-ID: <576cebe9.6a56c20a.d4270.56b4@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85366:ef2b3feaab48 Date: 2016-06-24 10:13 +0200 http://bitbucket.org/pypy/pypy/changeset/ef2b3feaab48/ Log: called getattr on wrong instance diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -546,8 +546,8 @@ log.write(inputargs, operations, ops_offset=ops_offset) # legacy - if log.logger_ops: - log.logger_ops.log_loop(inputargs, operations, 0, "rewritten", + if logger.logger_ops: + logger.logger_ops.log_loop(inputargs, operations, 0, "rewritten", name=loopname, ops_offset=ops_offset) self.fixup_target_tokens(rawstart) @@ -620,8 +620,8 @@ logger.log_patch_guard(descr_number, rawstart) # legacy - if log.logger_ops: - log.logger_ops.log_bridge(inputargs, operations, "rewritten", + if logger.logger_ops: + logger.logger_ops.log_bridge(inputargs, operations, "rewritten", faildescr, ops_offset=ops_offset) self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) From pypy.commits at gmail.com Fri Jun 24 04:42:15 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 24 Jun 2016 01:42:15 -0700 (PDT) Subject: [pypy-commit] pypy default: remove tests for object size; not really clear what is it meant to test Message-ID: <576cf267.4a79c20a.79161.66f7@mx.google.com> Author: Matti Picus Branch: Changeset: r85367:746d0c054448 Date: 2016-06-24 11:38 +0300 http://bitbucket.org/pypy/pypy/changeset/746d0c054448/ Log: remove tests for object size; not really clear what is it meant to test diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -17,24 +17,10 @@ """ PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); int result = 0; - size_t expected_size; if(PyByteArray_Size(s) == 12) { result = 1; } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*3; - #elif defined Py_DEBUG - expected_size = 64; - #else - expected_size = 48; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,23 +25,8 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyString_FromString("Hello world"); - int result; - size_t expected_size; + int result = PyString_Size(s); - result = PyString_Size(s); - - #ifdef PYPY_VERSION - expected_size = 48; - #elif defined Py_DEBUG - expected_size = 53; - #else - expected_size = 37; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyLong_FromLong(result); """), From pypy.commits at gmail.com Fri Jun 24 04:44:12 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 24 Jun 2016 01:44:12 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: 2 to 3 for merging Message-ID: <576cf2dc.c61f1c0a.a28ce.ffff9f07@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85368:1d0e93b54604 Date: 2016-06-24 10:26 +0300 http://bitbucket.org/pypy/pypy/changeset/1d0e93b54604/ Log: 2 to 3 for merging diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -1,6 +1,20 @@ #include "Python.h" #include "structmember.h" +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong + #define PyThing_FromStringAndSize PyUnicode_FromStringAndSize + #define PyThing_FromString PyUnicode_FromString + # defin PyThing_Check PyUnicode_Check + #define _PyThing_AsString _PyUnicode_AsString +#else + #define PyThing_FromStringAndSize PyString_FromStringAndSize + #define PyThing_FromString PyString_FromString + # defin PyThing_Check PyString_Check + #define _PyThing_AsString _PyString_AsString +#endif + typedef struct { PyObject_HEAD int foo; /* the context holder */ @@ -88,7 +102,7 @@ static PyObject * foo_get_name(PyObject *self, void *closure) { - return PyString_FromStringAndSize("Foo Example", 11); + return PyThing_FromStringAndSize("Foo Example", 11); } static PyObject * @@ -114,7 +128,7 @@ { PyObject *format; - format = PyString_FromString(""); + format = PyThing_FromString(""); if (format == NULL) return NULL; return format; } @@ -130,11 +144,11 @@ foo_setattro(fooobject *self, PyObject *name, PyObject *value) { char *name_str; - if (!PyString_Check(name)) { + if (!PyThing_Check(name)) { PyErr_SetObject(PyExc_AttributeError, name); return -1; } - name_str = PyString_AsString(name); + name_str = _PyThing_AsString(name); if (strcmp(name_str, "set_foo") == 0) { long v = PyInt_AsLong(value); @@ -652,6 +666,33 @@ {NULL, NULL} /* Sentinel */ }; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "foo", + "Module Doc", + -1, + foo_functions, + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_foo(void) + +#else + +#define INITERROR return NULL /* Initialize this module. */ #ifdef __GNUC__ @@ -662,8 +703,16 @@ PyMODINIT_FUNC initfoo(void) +#endif { - PyObject *m, *d; + PyObject *d; +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("foo", foo_functions); +#endif + if (module == NULL) + INITERROR; footype.tp_new = PyType_GenericNew; @@ -672,58 +721,59 @@ MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype2) < 0) - return; + INITERROR; if (PyType_Ready(&MetaType) < 0) - return; + INITERROR; if (PyType_Ready(&InitErrType) < 0) - return; + INITERROR; if (PyType_Ready(&SimplePropertyType) < 0) - return; + INITERROR; SimplePropertyType.tp_new = PyType_GenericNew; InitErrType.tp_new = PyType_GenericNew; CustomType.ob_type = &MetaType; if (PyType_Ready(&CustomType) < 0) - return; + INITERROR; UnicodeSubtype3.tp_flags = Py_TPFLAGS_DEFAULT; UnicodeSubtype3.tp_base = &UnicodeSubtype; UnicodeSubtype3.tp_bases = Py_BuildValue("(OO)", &UnicodeSubtype, &CustomType); if (PyType_Ready(&UnicodeSubtype3) < 0) - return; + INITERROR; TupleLike.tp_base = &PyTuple_Type; if (PyType_Ready(&TupleLike) < 0) return; - m = Py_InitModule("foo", foo_functions); - if (m == NULL) - return; - d = PyModule_GetDict(m); + + d = PyModule_GetDict(module); if (d == NULL) - return; + INITERROR; if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype3", (PyObject *) &UnicodeSubtype3) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "InitErrType", (PyObject *) &InitErrType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Property", (PyObject *) &SimplePropertyType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Custom", (PyObject *) &CustomType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) - return; + INITERROR; +#if PY_MAJOR_VERSION >=3 + return module; +#endif } diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -90,6 +90,7 @@ Py_DecRef(space, py_unicode.c_defenc) if py_unicode.c_str: lltype.free(py_unicode.c_str, flavor="raw") + from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) From pypy.commits at gmail.com Fri Jun 24 09:00:52 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 24 Jun 2016 06:00:52 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: typos Message-ID: <576d2f04.c7aec20a.6cac2.ffffc725@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85369:fba83d47f447 Date: 2016-06-24 15:40 +0300 http://bitbucket.org/pypy/pypy/changeset/fba83d47f447/ Log: typos diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -11,8 +11,8 @@ #else #define PyThing_FromStringAndSize PyString_FromStringAndSize #define PyThing_FromString PyString_FromString - # defin PyThing_Check PyString_Check - #define _PyThing_AsString _PyString_AsString + #define PyThing_Check PyString_Check + #define _PyThing_AsString PyString_AsString #endif typedef struct { @@ -692,7 +692,7 @@ #else -#define INITERROR return NULL +#define INITERROR return /* Initialize this module. */ #ifdef __GNUC__ From pypy.commits at gmail.com Fri Jun 24 09:25:51 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 24 Jun 2016 06:25:51 -0700 (PDT) Subject: [pypy-commit] pypy default: refactor _import_array to be more like upstream (used on pypy/numpy) Message-ID: <576d34df.cf2d1c0a.cecf5.132d@mx.google.com> Author: Matti Picus Branch: Changeset: r85370:a0105e0d00db Date: 2016-06-24 16:24 +0300 http://bitbucket.org/pypy/pypy/changeset/a0105e0d00db/ Log: refactor _import_array to be more like upstream (used on pypy/numpy) diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + From pypy.commits at gmail.com Fri Jun 24 09:25:53 2016 From: pypy.commits at gmail.com (mattip) Date: Fri, 24 Jun 2016 06:25:53 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: merge default into branch Message-ID: <576d34e1.8a40c20a.abf55.ffffdae3@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85371:a4fda7270791 Date: 2016-06-24 16:24 +0300 http://bitbucket.org/pypy/pypy/changeset/a4fda7270791/ Log: merge default into branch diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -17,24 +17,10 @@ """ PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); int result = 0; - size_t expected_size; if(PyByteArray_Size(s) == 12) { result = 1; } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*3; - #elif defined Py_DEBUG - expected_size = 64; - #else - expected_size = 48; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,23 +25,8 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyString_FromString("Hello world"); - int result; - size_t expected_size; + int result = PyString_Size(s); - result = PyString_Size(s); - - #ifdef PYPY_VERSION - expected_size = 48; - #elif defined Py_DEBUG - expected_size = 53; - #else - expected_size = 37; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyLong_FromLong(result); """), From pypy.commits at gmail.com Fri Jun 24 10:33:08 2016 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 24 Jun 2016 07:33:08 -0700 (PDT) Subject: [pypy-commit] pypy new-jit-log: mutable module global lists are not allowed in rpython Message-ID: <576d44a4.aa29c20a.f859c.fffff02f@mx.google.com> Author: Richard Plangger Branch: new-jit-log Changeset: r85372:30fc3a7cf4d3 Date: 2016-06-24 16:28 +0200 http://bitbucket.org/pypy/pypy/changeset/30fc3a7cf4d3/ Log: mutable module global lists are not allowed in rpython diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -13,7 +13,7 @@ from rpython.rtyper.annlowlevel import cast_instance_to_gcref, llhelper from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.jit.metainterp.debug import (DEBUG_COUNTER, LOOP_RUN_COUNTERS, +from rpython.jit.metainterp.debug import (DEBUG_COUNTER, debug_sd, flush_debug_counters) class GuardToken(object): @@ -333,7 +333,7 @@ self._call_assembler_patch_jmp(jmp_location) def get_loop_run_counters(self, index): - return LOOP_RUN_COUNTERS[index] + return debug_sd.loop_run_counters[index] @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations, tp, number): @@ -366,15 +366,16 @@ else: assert token struct.number = compute_unique_id(token) - LOOP_RUN_COUNTERS.append(struct) + debug_sd.loop_run_counters.append(struct) return struct def finish_once(self): if self._debug: # TODO remove the old logging system when jitlog is complete debug_start('jit-backend-counts') - for i in range(len(LOOP_RUN_COUNTERS)): - struct = LOOP_RUN_COUNTERS[i] + length = len(debug_sd.loop_run_counters) + for i in range(length): + struct = debug_sd.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number else: diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER from rpython.rlib import rgc -from rpython.jit.metainterp.debug import LOOP_RUN_COUNTERS +from rpython.jit.metainterp.debug import debug_sd from rpython.jit.backend.x86.assembler import Assembler386 from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls, xmm_reg_mgr_cls from rpython.jit.backend.x86.profagent import ProfileAgent @@ -116,8 +116,8 @@ def get_all_loop_runs(self): l = lltype.malloc(LOOP_RUN_CONTAINER, - len(LOOP_RUN_COUNTERS)) - for i, ll_s in enumerate(LOOP_RUN_COUNTERS): + len(debug_sd.loop_run_counters)) + for i, ll_s in enumerate(debug_sd.loop_run_counters): l[i].type = ll_s.type l[i].number = ll_s.number l[i].counter = ll_s.i diff --git a/rpython/jit/metainterp/debug.py b/rpython/jit/metainterp/debug.py --- a/rpython/jit/metainterp/debug.py +++ b/rpython/jit/metainterp/debug.py @@ -5,7 +5,11 @@ # forever, just because we want to report them at the end # of the process -LOOP_RUN_COUNTERS = [] +class DebugStaticData(object): + def __init__(self): + self.loop_run_counters = [] + +debug_sd = DebugStaticData() DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point @@ -16,8 +20,9 @@ def flush_debug_counters(): # this is always called, the jitlog knows if it is enabled - for i in range(len(LOOP_RUN_COUNTERS)): - struct = LOOP_RUN_COUNTERS[i] + length = len(debug_sd.loop_run_counters) + for i in range(length): + struct = debug_sd.loop_run_counters[i] _log_jit_counter(struct) # reset the counter, flush in a later point in time will # add up the counters! From pypy.commits at gmail.com Fri Jun 24 10:51:37 2016 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 24 Jun 2016 07:51:37 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: some explicit tests for virtual state with guard_compatible matching Message-ID: <576d48f9.6a25c20a.9c02e.ffffff78@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85373:574dea94736c Date: 2016-06-24 15:46 +0200 http://bitbucket.org/pypy/pypy/changeset/574dea94736c/ Log: some explicit tests for virtual state with guard_compatible matching diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -471,7 +471,6 @@ def test_guard_compatible(self): from rpython.jit.metainterp.compatible import CompatibilityCondition value1 = info.PtrInfo() - ptr = "fakeptr" value1._compatibility_conditions = CompatibilityCondition( ConstPtr(self.myptr)) box = InputArgRef() @@ -483,14 +482,10 @@ """ self.compare(guards, expected, [box]) - def test_guard_compatible_with_conditions(self): + def make_ccond(self): from rpython.jit.metainterp.compatible import CompatibilityCondition optimizer = FakeOptimizer(self.cpu) - value1 = info.PtrInfo() - ptr = "fakeptr" - ccond = value1._compatibility_conditions = CompatibilityCondition( - ConstPtr(self.quasiptr)) - + ccond = CompatibilityCondition(ConstPtr(self.quasiptr)) # regular call op = ResOperation( rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr)], @@ -513,6 +508,13 @@ op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) + return ccond + + def test_info_make_guards_guard_compatible_with_conditions(self): + value1 = info.PtrInfo() + ccond = self.make_ccond() + value1._compatibility_conditions = ccond + box = InputArgRef() guards = [] value1.make_guards(box, guards, FakeOptimizer(self.cpu)) @@ -529,6 +531,29 @@ """ self.compare(guards, expected, [box]) + def test_virtualstate_guard_compatible(self): + value1 = info.PtrInfo() + ccond1 = self.make_ccond() + value1._compatibility_conditions = ccond1 + value2 = info.PtrInfo() + ccond2 = self.make_ccond() + value2._compatibility_conditions = ccond2 + + state1 = not_virtual(self.cpu, 'r', value1) + state2 = not_virtual(self.cpu, 'r', value2) + self.check_no_guards(state1, state2) + + cond = ccond1.conditions[:] + ccond1.conditions = [cond[0]] + self.check_no_guards(state1, state2) + + ccond1.conditions = [cond[1]] + self.check_no_guards(state1, state2) + + ccond1.conditions = [] + self.check_no_guards(state1, state2) + + def test_equal_inputargs(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) value = info.InstancePtrInfo(None, classbox) From pypy.commits at gmail.com Fri Jun 24 10:51:39 2016 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 24 Jun 2016 07:51:39 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: another merging test Message-ID: <576d48fb.e457c20a.4a1b0.1377@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85374:482c68485a89 Date: 2016-06-24 15:52 +0200 http://bitbucket.org/pypy/pypy/changeset/482c68485a89/ Log: another merging test diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -362,3 +362,43 @@ # trace, two bridges, a finish bridge self.check_trace_count(4) + def test_merge_switch_object(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + p1 = lltype.malloc(S) + p1.x = 1 + + p2 = lltype.malloc(S) + p2.x = 1 + + driver = jit.JitDriver(greens = [], reds = ['n', 'x', 'y']) + + class A(object): + pass + + c = A() + c.count = 0 + @jit.elidable_compatible() + def g(s, ignored): + c.count += 1 + return s.x + + def f(n, x, y): + while n > 0: + driver.jit_merge_point(n=n, x=x, y=y) + n -= g(x, "abc") + if n % 6 == 5: + n -= 2 + x, y = y, x + + def main(): + g(p1, "def") # make annotator not make argument constant + f(1000, p1, p2) + f(1000, p2, p1) + return c.count + + x = self.meta_interp(main, []) + + assert x < 30 + # trace, one bridge, a finish bridge + self.check_trace_count(3) + From pypy.commits at gmail.com Fri Jun 24 10:51:41 2016 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 24 Jun 2016 07:51:41 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: a test with virtualstate merging and quasi-immutables Message-ID: <576d48fd.571b1c0a.a3512.35b3@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85375:7add21f0e70a Date: 2016-06-24 16:51 +0200 http://bitbucket.org/pypy/pypy/changeset/7add21f0e70a/ Log: a test with virtualstate merging and quasi-immutables diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -395,6 +395,7 @@ def emit_condition(self, op, short, optimizer, const=None): from rpython.jit.metainterp.resoperation import rop, ResOperation + from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr # more mess fielddescr = self.fielddescr if fielddescr.is_pointer_field(): @@ -406,9 +407,15 @@ else: getfield_op = ResOperation( rop.GETFIELD_GC_I, [op], fielddescr) + if const is not None: + ref = const.getref_base() + qmutdescr = QuasiImmutDescr( + optimizer.cpu, ref, self.fielddescr, self.mutatefielddescr) + else: + qmutdescr = self.qmutdescr short.extend([ ResOperation( - rop.QUASIIMMUT_FIELD, [op], self.qmutdescr), + rop.QUASIIMMUT_FIELD, [op], qmutdescr), ResOperation( rop.GUARD_NOT_INVALIDATED, []), getfield_op]) @@ -416,15 +423,19 @@ PureCallCondition.emit_condition(self, op, short, optimizer, const) call_op = short[index] # puh, not pretty - args = call_op.getarglist() + args = call_op.getarglist()[:] if const is not None: + args[1] = const del optimizer.call_pure_results[args] assert call_op.opnum in ( rop.CALL_PURE_I, rop.CALL_PURE_R, rop.CALL_PURE_F, rop.CALL_PURE_N) call_op.setarg(2, getfield_op) if const is not None: - optimizer.call_pure_results[call_op.getarglist()] = self.res + ref = const.getref_base() + args[2] = QuasiImmutDescr._get_fieldvalue( + self.fielddescr, ref, optimizer.cpu) + optimizer.call_pure_results[args] = self.res def repr(self, argrepr="?"): addr = self.args[0].getaddr() diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -629,7 +629,8 @@ self._last_guard_op) else: op = self.store_final_boxes_in_guard(guard_op, pendingfields) - self._last_guard_op = op + if opnum == rop.GUARD_COMPATIBLE: + self._last_guard_op = op # XXX don't share the next one either # for unrolling for farg in op.getfailargs(): if farg: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -482,7 +482,7 @@ """ self.compare(guards, expected, [box]) - def make_ccond(self): + def make_ccond_info(self): from rpython.jit.metainterp.compatible import CompatibilityCondition optimizer = FakeOptimizer(self.cpu) ccond = CompatibilityCondition(ConstPtr(self.quasiptr)) @@ -507,13 +507,12 @@ copied_op, cond = ccond.prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) - - return ccond + value = info.PtrInfo() + value._compatibility_conditions = ccond + return value def test_info_make_guards_guard_compatible_with_conditions(self): - value1 = info.PtrInfo() - ccond = self.make_ccond() - value1._compatibility_conditions = ccond + value1 = self.make_ccond_info() box = InputArgRef() guards = [] @@ -532,12 +531,8 @@ self.compare(guards, expected, [box]) def test_virtualstate_guard_compatible(self): - value1 = info.PtrInfo() - ccond1 = self.make_ccond() - value1._compatibility_conditions = ccond1 - value2 = info.PtrInfo() - ccond2 = self.make_ccond() - value2._compatibility_conditions = ccond2 + value1 = self.make_ccond_info() + value2 = self.make_ccond_info() state1 = not_virtual(self.cpu, 'r', value1) state2 = not_virtual(self.cpu, 'r', value2) @@ -553,6 +548,26 @@ ccond1.conditions = [] self.check_no_guards(state1, state2) + def test_virtualstate_guard_compatible_make_guards(self): + value1 = self.make_ccond_info() + value2 = self.make_ccond_info() + + state1 = not_virtual(self.cpu, 'r', value1) + state2 = not_virtual(self.cpu, 'r', value2) + + + cond = ccond2.conditions[:] + ccond2.conditions = [cond[0]] + box = InputArgRef(self.nodeaddr) + self.guards(state1, state2) + + ccond2.conditions = [cond[1]] + self.check_no_guards(state1, state2) + + ccond2.conditions = [] + self.check_no_guards(state1, state2) + self.check_no_guards(state1, state2) + def test_equal_inputargs(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -402,3 +402,82 @@ # trace, one bridge, a finish bridge self.check_trace_count(3) + + def test_quasi_immutable_merge(self): + from rpython.rlib.objectmodel import we_are_translated + class C(object): + _immutable_fields_ = ['version?'] + + class Version(object): + def __init__(self, cls): + self.cls = cls + p1 = C() + p1.version = Version(p1) + p1.x = 1 + p2 = C() + p2.version = Version(p2) + p2.x = 1 + p3 = C() + p3.version = Version(p3) + p3.x = 3 + + driver = jit.JitDriver(greens = [], reds = ['n', 'x']) + + class Counter(object): + pass + + c = Counter() + c.count = 0 + @jit.elidable_compatible() + def g(cls, v): + if we_are_translated(): + c.count += 1 + return cls.x + + def f(n, x): + res = 0 + while n > 0: + driver.can_enter_jit(n=n, x=x) + driver.jit_merge_point(n=n, x=x) + x = jit.hint(x, promote_compatible=True) + v = x.version + res = g(x, x.version) + n -= res + if n % 11 == 5: + n -= 1 + return res + + def main(x): + res = f(100, p1) + assert res == 1 + res = f(100, p2) + assert res == 1 + res = f(100, p3) + assert res == 3 + # invalidate p1 or p2 + if x: + p1.x = 2 + p1.version = Version(p1) + res = f(100, p1) + assert res == 2 + p1.x = 1 + p1.version = Version(p1) + else: + p2.x = 2 + p2.version = Version(p2) + res = f(100, p2) + assert res == 2 + p2.x = 1 + p2.version = Version(p2) + return c.count + main(True) + main(False) + + x = self.meta_interp(main, [True]) + assert x < 30 + + x = self.meta_interp(main, [False]) + assert x < 30 + self.check_trace_count(7) + + From pypy.commits at gmail.com Fri Jun 24 13:40:41 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 24 Jun 2016 10:40:41 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Weakrefs, in-progress Message-ID: <576d7099.07ecc20a.c7f59.36ad@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85376:fbdd9dca6850 Date: 2016-06-24 19:41 +0200 http://bitbucket.org/pypy/pypy/changeset/fbdd9dca6850/ Log: Weakrefs, in-progress diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -284,7 +284,6 @@ requires=[('translation.split_gc_address_space', True), ('translation.jit', False), ('translation.gc', 'boehm'), - ("translation.rweakref", False), ('translation.thread', False), ('translation.continuation', False)]), ]) diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -43,11 +43,14 @@ self.malloc_varsize_ptr = self.inittime_helper( ll_malloc_varsize, [lltype.Signed]*4, llmemory.GCREF, inline=False) if self.translator.config.translation.rweakref: + (ll_weakref_create, ll_weakref_deref, + self.WEAKLINK, self.convert_weakref_to + ) = build_weakref(self.translator.config) self.weakref_create_ptr = self.inittime_helper( ll_weakref_create, [llmemory.Address], llmemory.WeakRefPtr, inline=False) self.weakref_deref_ptr = self.inittime_helper( - ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address) + ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.GCREF) if not translator.config.translation.reverse_debugger: def ll_identityhash(addr): @@ -125,10 +128,10 @@ def gct_weakref_create(self, hop): v_instance, = hop.spaceop.args - v_addr = hop.genop("cast_ptr_to_adr", [v_instance], - resulttype=llmemory.Address) + v_gcref = hop.genop("cast_opaque_ptr", [v_instance], + resulttype=llmemory.GCREF) v_wref = hop.genop("direct_call", - [self.weakref_create_ptr, v_addr], + [self.weakref_create_ptr, v_gcref], resulttype=llmemory.WeakRefPtr) hop.cast_result(v_wref) @@ -140,10 +143,10 @@ def gct_weakref_deref(self, hop): v_wref, = hop.spaceop.args - v_addr = hop.genop("direct_call", - [self.weakref_deref_ptr, v_wref], - resulttype=llmemory.Address) - hop.cast_result(v_addr) + v_gcref = hop.genop("direct_call", + [self.weakref_deref_ptr, v_wref], + resulttype=llmemory.GCREF) + hop.cast_result(v_gcref) def gct_gc_writebarrier_before_copy(self, hop): # no write barrier needed @@ -180,32 +183,50 @@ # disappearing link. We don't have to hide the link's value with # HIDE_POINTER(), because we explicitly use GC_MALLOC_ATOMIC(). -WEAKLINK = lltype.FixedSizeArray(llmemory.Address, 1) -sizeof_weakreflink = llmemory.sizeof(WEAKLINK) -empty_weaklink = lltype.malloc(WEAKLINK, immortal=True) -empty_weaklink[0] = llmemory.NULL +def build_weakref(config): + revdb = config.translation.reverse_debugger + if not revdb: + WEAKLINK = lltype.Struct('WEAKLINK', + ('addr', llmemory.Address)) + else: + WEAKLINK = lltype.Struct('REVDB_WEAKLINK', + ('addr', llmemory.Address), + ('off_prev', lltype.SignedLongLong)) + sizeof_weakreflink = llmemory.sizeof(WEAKLINK) + empty_weaklink = lltype.malloc(WEAKLINK, immortal=True, zero=True) -def ll_weakref_create(targetaddr): - link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) - if not link: - raise MemoryError - plink = llmemory.cast_adr_to_ptr(link, lltype.Ptr(WEAKLINK)) - plink[0] = targetaddr - llop.boehm_disappearing_link(lltype.Void, link, targetaddr) - return llmemory.cast_ptr_to_weakrefptr(plink) + def ll_weakref_create(target_gcref): + if revdb: + plink = llop.revdb_weakref_create(lltype.Ptr(WEAKLINK), + target_gcref) + else: + link = llop.boehm_malloc_atomic(llmemory.Address, + sizeof_weakreflink) + if not link: + raise MemoryError + plink = llmemory.cast_adr_to_ptr(link, lltype.Ptr(WEAKLINK)) + plink.addr = llmemory.cast_ptr_to_adr(target_gcref) + llop.boehm_disappearing_link(lltype.Void, link, target_gcref) + return llmemory.cast_ptr_to_weakrefptr(plink) -def ll_weakref_deref(wref): - plink = llmemory.cast_weakrefptr_to_ptr(lltype.Ptr(WEAKLINK), wref) - return plink[0] + def ll_weakref_deref(wref): + plink = llmemory.cast_weakrefptr_to_ptr(lltype.Ptr(WEAKLINK), wref) + if revdb: + result = llop.revdb_weakref_deref(llmemory.GCREF, plink) + else: + result = llmemory.cast_adr_to_ptr(plink.addr, llmemory.GCREF) + return result -def convert_weakref_to(targetptr): - # Prebuilt weakrefs don't really need to be weak at all, - # but we need to emulate the structure expected by ll_weakref_deref(). - # This is essentially the same code as in ll_weakref_create(), but I'm - # not sure trying to share it is worth the hassle... - if not targetptr: - return empty_weaklink - else: - plink = lltype.malloc(WEAKLINK, immortal=True) - plink[0] = llmemory.cast_ptr_to_adr(targetptr) - return plink + def convert_weakref_to(targetptr): + # Prebuilt weakrefs don't really need to be weak at all, + # but we need to emulate the structure expected by ll_weakref_deref(). + # This is essentially the same code as in ll_weakref_create(), but I'm + # not sure trying to share it is worth the hassle... + if not targetptr: + return empty_weaklink + else: + plink = lltype.malloc(WEAKLINK, immortal=True, zero=True) + plink.addr = llmemory.cast_ptr_to_adr(targetptr) + return plink + + return ll_weakref_create, ll_weakref_deref, WEAKLINK, convert_weakref_to diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -572,6 +572,8 @@ 'revdb_get_unique_id': LLOp(sideeffects=False), 'revdb_watch_save_state': LLOp(), 'revdb_watch_restore_state': LLOp(), + 'revdb_weakref_create': LLOp(), + 'revdb_weakref_deref': LLOp(), } # ***** Run test_lloperation after changes. ***** diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -222,12 +222,10 @@ yield 'boehm_gc_startup_code();' def get_real_weakref_type(self): - from rpython.memory.gctransform import boehm - return boehm.WEAKLINK + return self.db.gctransformer.WEAKLINK def convert_weakref_to(self, ptarget): - from rpython.memory.gctransform import boehm - return boehm.convert_weakref_to(ptarget) + return self.db.gctransformer.convert_weakref_to(ptarget) def OP_GC__COLLECT(self, funcgen, op): return 'GC_gcollect();' diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1,6 +1,8 @@ #include "common_header.h" #include +#include #include +#include #include #include #include @@ -13,11 +15,15 @@ #include "forwarddecl.h" #include "preimpl.h" #include "src/rtyper.h" +#include "src/mem.h" #include "src-revdb/revdb_include.h" #define RDB_SIGNATURE "RevDB:" #define RDB_VERSION 0x00FF0001 +#define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) +#define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) + typedef struct { Signed version; @@ -170,6 +176,124 @@ return obj->h_hash; } +static int64_t recording_offset(void) +{ + off_t base_offset; + ssize_t extra_size = rpy_revdb.buf_p - rpy_rev_buffer; + + if (rpy_rev_buffer < 0) + return -1; + base_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR); + if (base_offset < 0) { + perror("lseek"); + exit(1); + } + return base_offset + extra_size; +} + +static void patch_prev_offset(int64_t offset, char old, char new) +{ + off_t base_offset; + if (rpy_rev_fileno < 0) + return; + base_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR); + if (base_offset < 0) { + perror("lseek"); + exit(1); + } + if (offset < base_offset) { + char got; + if (pread(rpy_rev_fileno, &got, 1, offset) != 1) { + fprintf(stderr, "can't read log position %lld for checking: %m\n", + (long long)offset); + exit(1); + } + if (got != old) { + fprintf(stderr, + "bad byte at log position %lld (%d instead of %d)\n", + (long long)offset, got, old); + exit(1); + } + if (pwrite(rpy_rev_fileno, &new, 1, offset) != 1) { + fprintf(stderr, "can't patch log position %lld\n", offset); + exit(1); + } + } + else { + ssize_t buffered_size = rpy_revdb.buf_p - rpy_rev_buffer; + int64_t buf_ofs = offset - base_offset; + if (buf_ofs >= buffered_size) { + fprintf(stderr, "invalid patch position %lld\n", + (long long)offset); + exit(1); + } + if (rpy_rev_buffer[buf_ofs] != old) { + fprintf(stderr, + "bad byte at log position %lld (%d instead of %d)\n", + (long long)offset, rpy_rev_buffer[buf_ofs], old); + exit(1); + } + rpy_rev_buffer[buf_ofs] = new; + } +} + +RPY_EXTERN +void *rpy_reverse_db_weakref_create(void *target) +{ + /* see comments in ../test/test_weak.py */ + struct pypy_REVDB_WEAKLINK0 *r; + r = malloc(sizeof(struct pypy_REVDB_WEAKLINK0)); + if (!r) { + fprintf(stderr, "out of memory for a weakref\n"); + exit(1); + } + r->re_addr = target; + r->re_off_prev = 0; + + if (!flag_io_disabled) { + char alive; + /* Emit WEAKREF_AFTERWARDS_DEAD, but remember where we emit it. + If we deref the weakref and it is still alive, we will patch + it with WEAKREF_AFTERWARDS_ALIVE. */ + if (!RPY_RDB_REPLAY) + r->re_off_prev = recording_offset(); + + RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); + + if (!RPY_RDB_REPLAY) { + OP_BOEHM_DISAPPEARING_LINK(r, target, /*nothing*/); + } + else if (alive == WEAKREF_AFTERWARDS_DEAD) + r->re_addr = NULL; + } + return r; +} + +RPY_EXTERN +void *rpy_reverse_db_weakref_deref(void *weakref) +{ + struct pypy_REVDB_WEAKLINK0 *r = (struct pypy_REVDB_WEAKLINK0 *)weakref; + void *result = r->re_addr; + if (result && !flag_io_disabled) { + char alive; + if (!RPY_RDB_REPLAY) { + if (r->re_off_prev <= 0) { + fprintf(stderr, "bug in weakrefs: bad previous offset %lld\n", + (long long)r->re_off_prev); + exit(1); + } + patch_prev_offset(r->re_off_prev, WEAKREF_AFTERWARDS_DEAD, + WEAKREF_AFTERWARDS_ALIVE); + r->re_off_prev = recording_offset(); + } + RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); + + if (RPY_RDB_REPLAY && alive == WEAKREF_AFTERWARDS_DEAD) + r->re_addr = NULL; + } + return result; +} + /* ------------------------------------------------------------ */ /* Replaying mode */ diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -107,6 +107,12 @@ #define OP_REVDB_WATCH_RESTORE_STATE(any_watch_point, r) \ rpy_reverse_db_watch_restore_state(any_watch_point) +#define OP_REVDB_WEAKREF_CREATE(target, r) \ + r = rpy_reverse_db_weakref_create(target) + +#define OP_REVDB_WEAKREF_DEREF(weakref, r) \ + r = rpy_reverse_db_weakref_deref(weakref) + RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size, const char *file, int line); @@ -119,5 +125,7 @@ RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object); RPY_EXTERN void rpy_reverse_db_watch_save_state(void); RPY_EXTERN void rpy_reverse_db_watch_restore_state(bool_t any_watch_point); +RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target); +RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/src-revdb/revdb_preinclude.h b/rpython/translator/revdb/src-revdb/revdb_preinclude.h --- a/rpython/translator/revdb/src-revdb/revdb_preinclude.h +++ b/rpython/translator/revdb/src-revdb/revdb_preinclude.h @@ -6,4 +6,5 @@ int64_t arg1; int64_t arg2; int64_t arg3; + /* char extra[extra_size]; */ } rpy_revdb_command_t; diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -64,7 +64,6 @@ t = Translation(entry_point, None, gc="boehm") self.t = t t.config.translation.reverse_debugger = True - t.config.translation.rweakref = False t.config.translation.lldebug0 = True if withsmallfuncsets is not None: t.config.translation.withsmallfuncsets = withsmallfuncsets @@ -91,11 +90,14 @@ return RDB(self.rdbname, map(str, expected_argv)) -class TestRecording(object): +class BaseRecordingTests(object): compile = compile run = run fetch_rdb = fetch_rdb + +class TestRecording(BaseRecordingTests): + def test_simple(self): def main(argv): print argv[1:] diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py new file mode 100644 --- /dev/null +++ b/rpython/translator/revdb/test/test_weak.py @@ -0,0 +1,68 @@ +import weakref +from rpython.translator.revdb.test.test_basic import BaseRecordingTests + + +# Weakrefs: implemented so that the log file contains one byte +# "afterwards_alive" or "afterwards_dead" at the point where the +# weakref is created, and similarly one such byte at every point where +# the weakref is dereferenced. At every point, the weakref is _alive_ +# at the moment; but the byte tells whether it should stay alive until +# the _next_ point or not. Done by always emitting "afterwards_dead" +# in the log, and patching that to "afterwards_alive" if later we find +# a deref() where the weakref is still alive. (If a deref() finds the +# weakref dead, it doesn't do any recording or patching; it simply +# leaves the previous "afterwards_dead" in place.) + + +WEAKREF_AFTERWARDS_DEAD = chr(0xf2) +WEAKREF_AFTERWARDS_ALIVE = chr(0xeb) + + +class TestRecordingWeakref(BaseRecordingTests): + + def test_weakref_create(self): + class X: + pass + class Glob: + pass + glob = Glob() + def main(argv): + glob.r1 = weakref.ref(X()) + glob.r2 = weakref.ref(X()) + glob.r3 = weakref.ref(X()) + return 9 + self.compile(main, [], backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + # find the extra WEAKREF_DEAD + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD + x = rdb.next('q'); assert x == 0 # number of stop points + assert rdb.done() + + def test_weakref_deref_nondead(self): + class X: + pass + class Glob: + pass + glob = Glob() + def main(argv): + x1 = X(); x2 = X() + r1 = weakref.ref(x1) + r2 = weakref.ref(x2) + assert r1() is x1 + assert r2() is x2 + assert r1() is x1 + return 9 + self.compile(main, [], backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + # find the five WEAKREF_xxx + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r1=ref(x1) + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r2=ref(x2) + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r1() + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD # r2() + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD # r1() + x = rdb.next('q'); assert x == 0 # number of stop points + assert rdb.done() From pypy.commits at gmail.com Fri Jun 24 16:01:54 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 24 Jun 2016 13:01:54 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Some fixes to weakrefs Message-ID: <576d91b2.c91d1c0a.1cad7.ffff988c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85377:86a0d99713cd Date: 2016-06-24 22:03 +0200 http://bitbucket.org/pypy/pypy/changeset/86a0d99713cd/ Log: Some fixes to weakrefs diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -115,7 +115,7 @@ if (filename && *filename) { putenv("PYPYRDB="); - rpy_rev_fileno = open(filename, O_WRONLY | O_CLOEXEC | + rpy_rev_fileno = open(filename, O_RDWR | O_CLOEXEC | O_CREAT | O_NOCTTY | O_TRUNC, 0600); if (rpy_rev_fileno < 0) { fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n", @@ -181,8 +181,8 @@ off_t base_offset; ssize_t extra_size = rpy_revdb.buf_p - rpy_rev_buffer; - if (rpy_rev_buffer < 0) - return -1; + if (rpy_rev_fileno < 0) + return 1; base_offset = lseek(rpy_rev_fileno, 0, SEEK_CUR); if (base_offset < 0) { perror("lseek"); @@ -242,7 +242,11 @@ { /* see comments in ../test/test_weak.py */ struct pypy_REVDB_WEAKLINK0 *r; - r = malloc(sizeof(struct pypy_REVDB_WEAKLINK0)); + if (!RPY_RDB_REPLAY) + r = GC_MALLOC_ATOMIC(sizeof(struct pypy_REVDB_WEAKLINK0)); + else + r = GC_MALLOC(sizeof(struct pypy_REVDB_WEAKLINK0)); + if (!r) { fprintf(stderr, "out of memory for a weakref\n"); exit(1); @@ -261,10 +265,23 @@ RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); if (!RPY_RDB_REPLAY) { - OP_BOEHM_DISAPPEARING_LINK(r, target, /*nothing*/); + OP_BOEHM_DISAPPEARING_LINK(&r->re_addr, target, /*nothing*/); } - else if (alive == WEAKREF_AFTERWARDS_DEAD) - r->re_addr = NULL; + else { + /* replaying: we don't make the weakref actually weak at all, + but instead we always know if we're going to need the + weakref value later or not */ + switch (alive) { + case WEAKREF_AFTERWARDS_DEAD: + r->re_addr = NULL; + break; + case WEAKREF_AFTERWARDS_ALIVE: + break; + default: + fprintf(stderr, "bad weakref_create byte in log\n"); + exit(1); + } + } } return r; } @@ -288,8 +305,18 @@ } RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); - if (RPY_RDB_REPLAY && alive == WEAKREF_AFTERWARDS_DEAD) - r->re_addr = NULL; + if (RPY_RDB_REPLAY) { + switch (alive) { + case WEAKREF_AFTERWARDS_DEAD: + r->re_addr = NULL; + break; + case WEAKREF_AFTERWARDS_ALIVE: + break; + default: + fprintf(stderr, "bad weakref_deref byte in log\n"); + exit(1); + } + } } return result; } diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -221,12 +221,14 @@ s2.close() self.subproc = subproc child = ReplayProcess(subproc.pid, s1) - child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, 3) + child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, + self.expected_stop_points) child.expect(ANSWER_READY, 1, Ellipsis) return child class TestSimpleInterpreter(InteractiveTests): + expected_stop_points = 3 def setup_class(cls): def main(argv): diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -1,5 +1,10 @@ -import weakref +import weakref, gc +from rpython.rlib import revdb +from rpython.rlib.debug import debug_print +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.translator.revdb.message import * from rpython.translator.revdb.test.test_basic import BaseRecordingTests +from rpython.translator.revdb.test.test_basic import InteractiveTests # Weakrefs: implemented so that the log file contains one byte @@ -11,7 +16,7 @@ # in the log, and patching that to "afterwards_alive" if later we find # a deref() where the weakref is still alive. (If a deref() finds the # weakref dead, it doesn't do any recording or patching; it simply -# leaves the previous "afterwards_dead" in place.) +# leaves the previous already-written "afterwards_dead" byte.) WEAKREF_AFTERWARDS_DEAD = chr(0xf2) @@ -49,20 +54,77 @@ glob = Glob() def main(argv): x1 = X(); x2 = X() - r1 = weakref.ref(x1) - r2 = weakref.ref(x2) - assert r1() is x1 - assert r2() is x2 - assert r1() is x1 + r1 = weakref.ref(x1) # (*) + r2 = weakref.ref(x2) # (*) + for i in range(8500): + assert r1() is x1 # (*) + assert r2() is x2 # (*) return 9 self.compile(main, [], backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) - # find the five WEAKREF_xxx - x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r1=ref(x1) - x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r2=ref(x2) - x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE # r1() - x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD # r2() - x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD # r1() + # find the 2 + 16998 first WEAKREF_xxx (all "(*)" but the last two) + for i in range(2 + 16998): + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE + for i in range(2): + x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() + + +class TestReplayingWeakref(InteractiveTests): + expected_stop_points = 1 + + def setup_class(cls): + from rpython.translator.revdb.test.test_basic import compile, run + + class X: + def __init__(self, s): + self.s = s + prebuilt = X('prebuilt') + + def make(s): + lst = [prebuilt] + [X(c) for c in s] + keepalive = lst[-1] + return [weakref.ref(x) for x in lst], keepalive + + def main(argv): + lst, keepalive = make(argv[0]) + expected = ['prebuilt'] + [c for c in argv[0]] + dead = [False] * len(lst) + for j in range(17000): + outp = [] + for i in range(len(lst)): + v = lst[i]() + debug_print(v) + if dead[i]: + assert v is None + elif v is None: + outp.append('') + dead[i] = True + else: + outp.append(v.s) + assert v.s == expected[i] + print ''.join(outp) + if (j % 1000) == 999: + debug_print('============= COLLECT ===========') + gc.collect() + debug_print('------ done', j, '.') + assert not dead[0] + assert not dead[-1] + keepalive_until_here(keepalive) + revdb.stop_point() + return 9 + compile(cls, main, [], backendopt=False) + output = run(cls, '') + lines = output.splitlines() + assert lines[-1].startswith('prebuilt') and lines[-1].endswith( + str(cls.exename)[-1]) + assert (len(lines[-1]) + output.count('') == + len('prebuilt') + len(str(cls.exename))) + + def test_replaying_weakref(self): + child = self.replay() + # the asserts are replayed; if we get here it means they passed again + child.send(Message(CMD_FORWARD, 1)) + child.expect(ANSWER_AT_END) From pypy.commits at gmail.com Fri Jun 24 16:10:08 2016 From: pypy.commits at gmail.com (arigo) Date: Fri, 24 Jun 2016 13:10:08 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Fix all tests Message-ID: <576d93a0.c445c20a.78ba3.6207@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85378:50ca83c04dca Date: 2016-06-24 22:11 +0200 http://bitbucket.org/pypy/pypy/changeset/50ca83c04dca/ Log: Fix all tests diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -189,6 +189,7 @@ WEAKLINK = lltype.Struct('WEAKLINK', ('addr', llmemory.Address)) else: + # keep in sync with 'struct WEAKLINK' in revdb.c WEAKLINK = lltype.Struct('REVDB_WEAKLINK', ('addr', llmemory.Address), ('off_prev', lltype.SignedLongLong)) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -76,7 +76,7 @@ return ''.join(pieces) def send(self, msg): - print 'SENT:', self.pid, msg + #print 'SENT:', self.pid, msg binary = struct.pack("iIqqq", msg.cmd, len(msg.extra), msg.arg1, msg.arg2, msg.arg3) self.control_socket.sendall(binary + msg.extra) @@ -86,7 +86,7 @@ cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary) extra = self._recv_all(size) msg = Message(cmd, arg1, arg2, arg3, extra) - print 'RECV:', self.pid, msg + #print 'RECV:', self.pid, msg return msg def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""): diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -237,15 +237,21 @@ } } +/* keep in sync with 'REVDB_WEAKLINK' in rpython.memory.gctransform.boehm */ +struct WEAKLINK { + void *re_addr; + long long re_off_prev; +}; + RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target) { /* see comments in ../test/test_weak.py */ - struct pypy_REVDB_WEAKLINK0 *r; + struct WEAKLINK *r; if (!RPY_RDB_REPLAY) - r = GC_MALLOC_ATOMIC(sizeof(struct pypy_REVDB_WEAKLINK0)); + r = GC_MALLOC_ATOMIC(sizeof(struct WEAKLINK)); else - r = GC_MALLOC(sizeof(struct pypy_REVDB_WEAKLINK0)); + r = GC_MALLOC(sizeof(struct WEAKLINK)); if (!r) { fprintf(stderr, "out of memory for a weakref\n"); @@ -289,7 +295,7 @@ RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref) { - struct pypy_REVDB_WEAKLINK0 *r = (struct pypy_REVDB_WEAKLINK0 *)weakref; + struct WEAKLINK *r = (struct WEAKLINK *)weakref; void *result = r->re_addr; if (result && !flag_io_disabled) { char alive; diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -271,6 +271,7 @@ class TestDebugCommands(InteractiveTests): + expected_stop_points = 3 def setup_class(cls): # From pypy.commits at gmail.com Sat Jun 25 14:05:02 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sat, 25 Jun 2016 11:05:02 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: use 6 parameters for ast.arguments in lambdef Message-ID: <576ec7ce.6a56c20a.67051.2e1b@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85379:0b9ea720dd07 Date: 2016-06-25 20:04 +0200 http://bitbucket.org/pypy/pypy/changeset/0b9ea720dd07/ Log: use 6 parameters for ast.arguments in lambdef diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -844,7 +844,7 @@ def handle_lambdef(self, lambdef_node): expr = self.handle_expr(lambdef_node.get_child(-1)) if lambdef_node.num_children() == 3: - args = ast.arguments(None, None, None, None, None, None, None, None) + args = ast.arguments(None, None, None, None, None, None) else: args = self.handle_arguments(lambdef_node.get_child(1)) return ast.Lambda(args, expr, lambdef_node.get_lineno(), lambdef_node.get_column()) From pypy.commits at gmail.com Sat Jun 25 14:13:20 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 25 Jun 2016 11:13:20 -0700 (PDT) Subject: [pypy-commit] buildbot default: limit log files Message-ID: <576ec9c0.0476c20a.ee28.ffffbe12@mx.google.com> Author: Matti Picus Branch: Changeset: r1011:1d84a7614225 Date: 2016-06-25 20:12 +0200 http://bitbucket.org/pypy/buildbot/changeset/1d84a7614225/ Log: limit log files diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -547,4 +547,6 @@ # "buildmaster","XndZopHM"), 'buildbotURL': 'http://buildbot.pypy.org/', # with a trailing '/'! 'projectURL': 'http://pypy.org/', - 'projectName': 'PyPy'} + 'projectName': 'PyPy', + 'logMaxSize': 5*1024*1204, # 5M + } From pypy.commits at gmail.com Sun Jun 26 04:23:33 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 01:23:33 -0700 (PDT) Subject: [pypy-commit] pypy default: Trying to implement Boehm support for rgc.FinalizerQueue Message-ID: <576f9105.07ecc20a.9e772.39c7@mx.google.com> Author: Armin Rigo Branch: Changeset: r85380:827cb0a67372 Date: 2016-06-26 10:24 +0200 http://bitbucket.org/pypy/pypy/changeset/827cb0a67372/ Log: Trying to implement Boehm support for rgc.FinalizerQueue diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -1,6 +1,7 @@ from rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from rpython.memory.gctransform.support import (get_rtti, - _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor) + _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor, + ll_report_finalizer_error) from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.flowspace.model import Constant from rpython.rtyper.lltypesystem.lloperation import llop @@ -58,6 +59,9 @@ self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() + self.finalizer_triggers = [] + self.finalizer_queue_indexes = {} # {fq: index} + def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): # XXX same behavior for zero=True: in theory that's wrong if TYPE._is_atomic(): @@ -115,6 +119,39 @@ self.finalizer_funcptrs[TYPE] = fptr return fptr + def get_finalizer_queue_index(self, hop): + fq_tag = hop.spaceop.args[0].value + assert 'FinalizerQueue TAG' in fq_tag.expr + fq = fq_tag.default + try: + index = self.finalizer_queue_indexes[fq] + except KeyError: + index = len(self.finalizer_queue_indexes) + assert index == len(self.finalizer_triggers) + # + def ll_finalizer_trigger(): + try: + fq.finalizer_trigger() + except Exception as e: + ll_report_finalizer_error(e) + ll_trigger = self.annotate_finalizer(ll_finalizer_trigger, [], + lltype.Void) + self.finalizer_triggers.append(ll_trigger) + self.finalizer_queue_indexes[fq] = index + return index + + def gct_gc_fq_register(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + v_ptr = hop.spaceop.args[1] + hop.genop("boehm_fq_register", [c_index, v_ptr]) + + def gct_gc_fq_next_dead(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + hop.genop("boehm_fq_next_dead", [c_index], + resultvar = hop.spaceop.result) + def gct_weakref_create(self, hop): v_instance, = hop.spaceop.args v_addr = hop.genop("cast_ptr_to_adr", [v_instance], diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -212,6 +212,24 @@ compile_extra=['-DPYPY_USING_BOEHM_GC'], )) + gct = self.db.gctransformer + gct.finalizer_triggers = tuple(gct.finalizer_triggers) # stop changing + sourcelines = [''] + for trig in gct.finalizer_triggers: + sourcelines.append('RPY_EXTERN void %s(void);' % ( + self.db.get(trig),)) + sourcelines.append('') + sourcelines.append('void (*boehm_fq_trigger[])(void) = {') + for trig in gct.finalizer_triggers: + sourcelines.append('\t%s,' % (self.db.get(trig),)) + sourcelines.append('\tNULL') + sourcelines.append('};') + sourcelines.append('struct boehm_fq_s *boehm_fq_queues[%d];' % ( + len(gct.finalizer_triggers),)) + sourcelines.append('') + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=['\n'.join(sourcelines)])) + return eci def gc_startup_code(self): diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -73,9 +73,17 @@ #ifdef PYPY_USING_BOEHM_GC +struct boehm_fq_s { + void *obj; + struct boehm_fq_s *next; +}; +RPY_EXTERN void (*boehm_fq_trigger[])(void); + int boehm_gc_finalizer_lock = 0; void boehm_gc_finalizer_notifier(void) { + int i; + boehm_gc_finalizer_lock++; while (GC_should_invoke_finalizers()) { if (boehm_gc_finalizer_lock > 1) { @@ -86,6 +94,11 @@ } GC_invoke_finalizers(); } + + i = 0; + while (boehm_fq_trigger[i]) + boehm_fq_trigger[i++](); + boehm_gc_finalizer_lock--; } @@ -100,6 +113,28 @@ GC_finalize_on_demand = 1; GC_set_warn_proc(mem_boehm_ignore); } + +void boehm_fq_callback(void *obj, void *rawfqueue) +{ + struct boehm_fq_s **fqueue = rawfqueue; + struct boehm_fq_s *node = GC_malloc(sizeof(void *) * 2); + if (!node) + return; /* ouch, too bad */ + node->obj = obj; + node->next = *fqueue; + *fqueue = node; +} + +void *boehm_fq_next_dead(struct boehm_fq_s **fqueue) +{ + struct boehm_fq_s *node = *fqueue; + if (node != NULL) { + *fqueue = node->next; + return node->obj; + } + else + return NULL; +} #endif /* BOEHM GC */ diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -104,13 +104,21 @@ RPY_EXTERN int boehm_gc_finalizer_lock; RPY_EXTERN void boehm_gc_startup_code(void); RPY_EXTERN void boehm_gc_finalizer_notifier(void); +struct boehm_fq_s; +RPY_EXTERN struct boehm_fq_s *boehm_fq_queues[]; +RPY_EXTERN void (*boehm_fq_trigger[])(void); +RPY_EXTERN void boehm_fq_callback(void *, void *); +RPY_EXTERN void *boehm_fq_next_dead(struct boehm_fq_s **); #define OP_GC__DISABLE_FINALIZERS(r) boehm_gc_finalizer_lock++ #define OP_GC__ENABLE_FINALIZERS(r) (boehm_gc_finalizer_lock--, \ boehm_gc_finalizer_notifier()) -#define OP_GC_FQ_REGISTER(tag, obj, r) /* ignored so far */ -#define OP_GC_FQ_NEXT_DEAD(tag, r) (r = NULL) +#define OP_BOEHM_FQ_REGISTER(tagindex, obj, r) \ + GC_REGISTER_FINALIZER(obj, boehm_fq_callback, \ + boehm_fq_queues + tagindex, NULL, NULL) +#define OP_BOEHM_FQ_NEXT_DEAD(tagindex, r) \ + r = boehm_fq_next_dead(boehm_fq_queues + tagindex) #endif /* PYPY_USING_BOEHM_GC */ diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -393,20 +393,36 @@ assert res[3] == compute_hash(d) assert res[4] == compute_hash(("Hi", None, (7.5, 2, d))) - def test_finalizer_queue_is_at_least_ignored(self): + def test_finalizer_queue(self): class A(object): - pass + def __init__(self, i): + self.i = i + class Glob: + triggered = 0 + glob = Glob() class FQ(rgc.FinalizerQueue): Class = A + triggered = 0 def finalizer_trigger(self): - debug.debug_print("hello!") # not called so far + glob.triggered += 1 fq = FQ() # def fn(): - fq.register_finalizer(A()) + for i in range(1000): + fq.register_finalizer(A(i)) rgc.collect() rgc.collect() - fq.next_dead() + if glob.triggered == 0: + print "not triggered!" + return 50 + seen = {} + for i in range(1000): + a = fq.next_dead() + assert a.i not in seen + seen[a.i] = True + if len(seen) < 500: + print "seen only %d!" % len(seen) + return 51 return 42 f = self.getcompiled(fn) From pypy.commits at gmail.com Sun Jun 26 08:49:56 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 26 Jun 2016 05:49:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix stackdeptherror by changing nsubargs -> nsubkwargs Message-ID: <576fcf74.46461c0a.35152.0a34@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85381:67f158f6b513 Date: 2016-06-26 14:49 +0200 http://bitbucket.org/pypy/pypy/changeset/67f158f6b513/ Log: Fix stackdeptherror by changing nsubargs -> nsubkwargs diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1197,7 +1197,7 @@ # Pack up any trailing keyword arguments. self.emit_op_arg(ops.BUILD_MAP,nseen) nsubkwargs += 1 - if nsubargs: + if nsubkwargs: call_type |= 2 if nsubkwargs > 1: # Pack it all up From pypy.commits at gmail.com Sun Jun 26 10:28:42 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 26 Jun 2016 07:28:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: ctx not needed in visit_starunpack Message-ID: <576fe69a.c5461c0a.265c8.2574@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85382:905965fb1e52 Date: 2016-06-26 16:28 +0200 http://bitbucket.org/pypy/pypy/changeset/905965fb1e52/ Log: ctx not needed in visit_starunpack diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1017,7 +1017,7 @@ ifexp.orelse.walkabout(self) self.use_next_block(end) - def _visit_starunpack(self, node, elts, ctx, single_op, inner_op, outer_op): + def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op): elt_count = len(elts) if elts else 0 seen_star = 0 elt_subitems = 0 @@ -1074,7 +1074,7 @@ if tup.ctx == ast.Store: self._visit_assignment(tup, tup.elts, tup.ctx) elif tup.ctx == ast.Load: - self._visit_starunpack(tup, tup.elts, tup.ctx, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK) + self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK) else: self.visit_sequence(tup.elts) @@ -1083,7 +1083,7 @@ if l.ctx == ast.Store: self._visit_assignment(l, l.elts, l.ctx) elif l.ctx == ast.Load: - self._visit_starunpack(l, l.elts, l.ctx, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK) + self._visit_starunpack(l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK) else: self.visit_sequence(l.elts) @@ -1121,7 +1121,7 @@ is_unpacking = 0 def visit_Set(self, s): - self._visit_starunpack(s, s.elts, s.ctx, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK) + self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK) def visit_Name(self, name): self.update_position(name.lineno) From pypy.commits at gmail.com Sun Jun 26 10:44:53 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 26 Jun 2016 07:44:53 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Changed wrong var name code -> call_type Message-ID: <576fea65.8999c20a.42037.ffff8f7e@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85383:9430fdd3c85c Date: 2016-06-26 16:35 +0200 http://bitbucket.org/pypy/pypy/changeset/9430fdd3c85c/ Log: Changed wrong var name code -> call_type diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1201,7 +1201,7 @@ call_type |= 2 if nsubkwargs > 1: # Pack it all up - function_pos = n + (code & 1) + nkw + 1 + function_pos = n + (call_type & 1) + nkw + 1 self.emit_op_arg(ops.BUILD_MAP_UNPACK_WITH_CALL, (nsubkwargs | (function_pos << 8))) assert n < 1<<8 From pypy.commits at gmail.com Sun Jun 26 14:01:40 2016 From: pypy.commits at gmail.com (raffael_t) Date: Sun, 26 Jun 2016 11:01:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement pyopcode build_list_unpack Message-ID: <57701884.c3a9c20a.4b461.6b37@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85384:a131597a62b3 Date: 2016-06-26 20:01 +0200 http://bitbucket.org/pypy/pypy/changeset/a131597a62b3/ Log: Implement pyopcode build_list_unpack diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1355,16 +1355,18 @@ #self.pushvalue(w_sum) def BUILD_LIST_UNPACK(self, itemcount, next_instr): - self.BUILD_LIST(itemcount, next_instr) - #w_sum = self.space.newlist() - #for i in range(itemcount, 0, -1): - # w_item = self.popvalue() - # #self.space.peek(i) - # self.space.call_method(w_sum, 'update', w_item) - ##while itemcount != 0: - ## self.popvalue() - ## itemcount -= 1 - #self.pushvalue(w_sum) + w_sum = [] + for i in range(itemcount, 0, -1): + #for i in range(0, itemcount): + #w_item = self.popvalue() + w_item = self.peekvalue(i-1) + items = self.space.fixedview(w_item) + w_sum.extend(items) + #w_sum.append(w_item) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(self.space.newlist(w_sum)) #TODO #get intersection, store as setentry From pypy.commits at gmail.com Sun Jun 26 16:21:02 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:21:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Hit a case where this incorrect ll2ctypes type is really wrong. Trying Message-ID: <5770392e.c7a81c0a.1be5a.ffffb541@mx.google.com> Author: Armin Rigo Branch: Changeset: r85385:a635618d95ba Date: 2016-06-26 14:39 +0200 http://bitbucket.org/pypy/pypy/changeset/a635618d95ba/ Log: Hit a case where this incorrect ll2ctypes type is really wrong. Trying to improve... diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -166,7 +166,19 @@ }) if '__int128_t' in rffi.TYPES: - _ctypes_cache[rffi.__INT128_T] = ctypes.c_longlong # XXX: Not right at all. But for some reason, It started by while doing JIT compile after a merge with default. Can't extend ctypes, because thats a python standard, right? + class c_int128(ctypes.Array): # based on 2 ulongs + _type_ = ctypes.c_uint64 + _length_ = 2 + @property + def value(self): + if sys.byteorder == 'little': + res = self[0] | (self[1] << 64) + else: + res = self[1] | (self[0] << 64) + if res >= (1 << 127): + res -= 1 << 128 + return res + _ctypes_cache[rffi.__INT128_T] = c_int128 # for unicode strings, do not use ctypes.c_wchar because ctypes # automatically converts arrays into unicode strings. diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -825,3 +825,13 @@ assert charp2str(p2) == "helLD" free_charp(p1) free_charp(p2) + +def test_sign_when_casting_uint_to_larger_int(): + from rpython.rtyper.lltypesystem import rffi + from rpython.rlib.rarithmetic import r_uint32, r_uint64 + # + value = 0xAAAABBBB + assert cast(lltype.SignedLongLong, r_uint32(value)) == value + if hasattr(rffi, '__INT128_T'): + value = 0xAAAABBBBCCCCDDDD + assert cast(rffi.__INT128_T, r_uint64(value)) == value From pypy.commits at gmail.com Sun Jun 26 16:21:08 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:21:08 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix Message-ID: <57703934.949a1c0a.c5036.0176@mx.google.com> Author: Armin Rigo Branch: Changeset: r85388:507eee5b4548 Date: 2016-06-26 22:15 +0200 http://bitbucket.org/pypy/pypy/changeset/507eee5b4548/ Log: Fix diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1838,7 +1838,7 @@ rem = (rem << SHIFT) | pin.widedigit(size) hi = llop.long2_floordiv(lltype.Signed, rem, n) pout.setdigit(size, hi) - rem -= hi * n + rem -= _widen_digit(hi) * n size -= 1 return rffi.cast(lltype.Signed, rem) From pypy.commits at gmail.com Sun Jun 26 16:21:10 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:21:10 -0700 (PDT) Subject: [pypy-commit] pypy default: Oops, we decided to use '__int128_t' instead of '__int128' Message-ID: <57703936.e643c20a.1b19c.2fc4@mx.google.com> Author: Armin Rigo Branch: Changeset: r85389:bc5641f92910 Date: 2016-06-26 22:22 +0200 http://bitbucket.org/pypy/pypy/changeset/bc5641f92910/ Log: Oops, we decided to use '__int128_t' instead of '__int128' diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -121,7 +121,7 @@ #define OP_LLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ r = (long long)(((unsigned long long)(x)) << (y)) #define OP_LLLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, 128); \ - r = (__int128)(((unsigned __int128)(x)) << (y)) + r = (__int128_t)(((__uint128_t)(x)) << (y)) #define OP_UINT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ r = (x) << (y) @@ -158,7 +158,7 @@ #define OP_CAST_UINT_TO_INT(x,r) r = (Signed)(x) #define OP_CAST_INT_TO_UINT(x,r) r = (Unsigned)(x) #define OP_CAST_INT_TO_LONGLONG(x,r) r = (long long)(x) -#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128)(x) +#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128_t)(x) #define OP_CAST_CHAR_TO_INT(x,r) r = (Signed)((unsigned char)(x)) #define OP_CAST_INT_TO_CHAR(x,r) r = (char)(x) #define OP_CAST_PTR_TO_INT(x,r) r = (Signed)(x) /* XXX */ From pypy.commits at gmail.com Sun Jun 26 16:21:04 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:21:04 -0700 (PDT) Subject: [pypy-commit] pypy default: Use custom assembler for divisions of 128 bits by 64 bits with a result Message-ID: <57703930.488e1c0a.7b621.07c9@mx.google.com> Author: Armin Rigo Branch: Changeset: r85386:800377eb1f02 Date: 2016-06-26 15:37 +0200 http://bitbucket.org/pypy/pypy/changeset/800377eb1f02/ Log: Use custom assembler for divisions of 128 bits by 64 bits with a result that fits 64 bits. It's hard to get this effect automatically while writing C. diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1827,6 +1827,8 @@ Divide bigint pin by non-zero digit n, storing quotient in pout, and returning the remainder. It's OK for pin == pout on entry. """ + from rpython.rtyper.lltypesystem.lloperation import llop + rem = _widen_digit(0) assert n > 0 and n <= MASK if not size: @@ -1834,7 +1836,7 @@ size -= 1 while size >= 0: rem = (rem << SHIFT) | pin.widedigit(size) - hi = rem // n + hi = llop.long2_floordiv(lltype.Signed, rem, n) pout.setdigit(size, hi) rem -= hi * n size -= 1 @@ -1924,6 +1926,7 @@ z._normalize() return z _muladd1._annspecialcase_ = "specialize:argtype(2)" + def _v_lshift(z, a, m, d): """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put * result in z[0:m], and return the d bits shifted out of the top. @@ -1961,6 +1964,8 @@ def _x_divrem(v1, w1): """ Unsigned bigint division with remainder -- the algorithm """ + from rpython.rtyper.lltypesystem.lloperation import llop + size_v = v1.numdigits() size_w = w1.numdigits() assert size_v >= size_w and size_w > 1 @@ -1991,6 +1996,7 @@ assert k > 0 a = rbigint([NULLDIGIT] * k, 1, k) + wm1s = w.digit(abs(size_w-1)) wm1 = w.widedigit(abs(size_w-1)) wm2 = w.widedigit(abs(size_w-2)) @@ -2008,7 +2014,7 @@ vtop = v.widedigit(j) assert vtop <= wm1 vv = (vtop << SHIFT) | v.widedigit(abs(j-1)) - q = vv / wm1 + q = llop.long2_floordiv(lltype.Signed, vv, wm1s) r = vv - wm1 * q while wm2 * q > ((r << SHIFT) | v.widedigit(abs(j-2))): q -= 1 diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -319,6 +319,9 @@ 'lllong_rshift': LLOp(canfold=True), # args (r_longlonglong, int) 'lllong_xor': LLOp(canfold=True), + 'long2_floordiv': LLOp(canfold=True), # (double-r_long, int) => int + # (all integers signed) + 'cast_primitive': LLOp(canfold=True), 'cast_bool_to_int': LLOp(canfold=True), 'cast_bool_to_uint': LLOp(canfold=True), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -16,7 +16,7 @@ 'bool': True, 'is_true':True} # global synonyms for some types -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, base_int from rpython.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong, r_longlonglong from rpython.rtyper.lltypesystem.llmemory import AddressAsInt @@ -733,6 +733,16 @@ assert isinstance(x, bool) return x +def op_long2_floordiv(x, y): + if lltype.typeOf(x) != lltype.Signed: + assert isinstance(x, base_int) + assert x.BITS == 2 * r_int.BITS + assert x.SIGNED + assert lltype.typeOf(y) is lltype.Signed + result = int(x) // y + assert result == intmask(result), "overflow in long2_floordiv" + return result + # ____________________________________________________________ def get_op_impl(opname): diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -106,3 +106,11 @@ #define PYPY_X86_CHECK_SSE2_DEFINED RPY_EXTERN void pypy_x86_check_sse2(void); #endif + + +#undef OP_LONG2_FLOORDIV +/* assumes that 'y' and 'r' fit in a signed word, + but 'x' takes up to two words */ +#define OP_LONG2_FLOORDIV(x, y, r) \ + __asm__("idiv %1" : "=a"(r) : \ + "r"((long)y), "A"((long long)x)); diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -6,3 +6,10 @@ asm volatile("rdtsc" : "=a"(_rax), "=d"(_rdx)); \ val = (_rdx << 32) | _rax; \ } while (0) + +#undef OP_LONG2_FLOORDIV +/* assumes that 'y' and 'r' fit in a signed word, + but 'x' takes up to two words */ +#define OP_LONG2_FLOORDIV(x, y, r) \ + __asm__("idiv %1" : "=a"(r) : \ + "r"((long)y), "a"((long)x), "d"((long)((x >> 32) >> 32))) diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -135,6 +135,7 @@ #define OP_LLONG_FLOORDIV(x,y,r) r = (x) / (y) #define OP_ULLONG_FLOORDIV(x,y,r) r = (x) / (y) #define OP_LLLONG_FLOORDIV(x,y,r) r = (x) / (y) +#define OP_LONG2_FLOORDIV(x,y,r) r = (x) / (y) /* modulus */ diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py --- a/rpython/translator/c/test/test_lltyped.py +++ b/rpython/translator/c/test/test_lltyped.py @@ -1,6 +1,7 @@ -import py +import py, sys, random from rpython.rtyper.lltypesystem.lltype import * from rpython.rtyper.lltypesystem import rffi +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator.c.test.test_genc import compile from rpython.tool.sourcetools import func_with_new_name @@ -1023,3 +1024,27 @@ assert fn(r_longlong(1)) == True assert fn(r_longlong(256)) == True assert fn(r_longlong(2**32)) == True + + def test_long2_floordiv(self): + def f(a, b): + return llop.long2_floordiv(Signed, a, b) + fn = self.getcompiled(f, [int, int]) + assert fn(100, 3) == 33 + # + if sys.maxint > 2**32: + HUGE = getattr(rffi, '__INT128_T', None) + bits = 128 + else: + HUGE = SignedLongLong + bits = 64 + if HUGE is not None: + def f(a, b, c): + ab = (rffi.cast(HUGE, a) << (bits//2)) | b + return llop.long2_floordiv(Signed, ab, c) + fn = self.getcompiled(f, [int, int, int]) + for i in range(100): + a = random.randrange(0, 10) + b = random.randrange(0, sys.maxint+1) + c = random.randrange(2*a+2, 25) + print a, b, c + assert fn(a, b, c) == ((a << (bits//2)) | b) // c From pypy.commits at gmail.com Sun Jun 26 16:21:06 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:21:06 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix Message-ID: <57703932.c99dc20a.c3ac5.0735@mx.google.com> Author: Armin Rigo Branch: Changeset: r85387:99ea9c77a44c Date: 2016-06-26 20:10 +0100 http://bitbucket.org/pypy/pypy/changeset/99ea9c77a44c/ Log: Fix diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -319,8 +319,8 @@ 'lllong_rshift': LLOp(canfold=True), # args (r_longlonglong, int) 'lllong_xor': LLOp(canfold=True), - 'long2_floordiv': LLOp(canfold=True), # (double-r_long, int) => int - # (all integers signed) + 'long2_floordiv': LLOp(canfold=True), # r = x / y: r and y Signed, + # x can have 2x the size 'cast_primitive': LLOp(canfold=True), 'cast_bool_to_int': LLOp(canfold=True), diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -111,6 +111,8 @@ #undef OP_LONG2_FLOORDIV /* assumes that 'y' and 'r' fit in a signed word, but 'x' takes up to two words */ -#define OP_LONG2_FLOORDIV(x, y, r) \ - __asm__("idiv %1" : "=a"(r) : \ - "r"((long)y), "A"((long long)x)); +#define OP_LONG2_FLOORDIV(x, y, r) do { \ + long ignored; \ + __asm__("idiv %2" : "=a"(r), "=d"(ignored) : \ + "r"((long)y), "A"((long long)x)); \ + } while (0) diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -10,6 +10,9 @@ #undef OP_LONG2_FLOORDIV /* assumes that 'y' and 'r' fit in a signed word, but 'x' takes up to two words */ -#define OP_LONG2_FLOORDIV(x, y, r) \ - __asm__("idiv %1" : "=a"(r) : \ - "r"((long)y), "a"((long)x), "d"((long)((x >> 32) >> 32))) +#define OP_LONG2_FLOORDIV(x, y, r) do { \ + long ignored; \ + __asm__("idiv %2" : "=a"(r), "=d"(ignored) : \ + "r"((long)y), "a"((long)x), \ + "d"((long)((x >> 32) >> 32))); \ + } while (0) From pypy.commits at gmail.com Sun Jun 26 16:26:21 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:26:21 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix, comment Message-ID: <57703a6d.06a61c0a.876f7.ffffef7a@mx.google.com> Author: Armin Rigo Branch: Changeset: r85390:469aff993fa6 Date: 2016-06-26 22:27 +0200 http://bitbucket.org/pypy/pypy/changeset/469aff993fa6/ Log: Fix, comment diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1964,8 +1964,6 @@ def _x_divrem(v1, w1): """ Unsigned bigint division with remainder -- the algorithm """ - from rpython.rtyper.lltypesystem.lloperation import llop - size_v = v1.numdigits() size_w = w1.numdigits() assert size_v >= size_w and size_w > 1 @@ -1996,7 +1994,6 @@ assert k > 0 a = rbigint([NULLDIGIT] * k, 1, k) - wm1s = w.digit(abs(size_w-1)) wm1 = w.widedigit(abs(size_w-1)) wm2 = w.widedigit(abs(size_w-2)) @@ -2014,7 +2011,7 @@ vtop = v.widedigit(j) assert vtop <= wm1 vv = (vtop << SHIFT) | v.widedigit(abs(j-1)) - q = llop.long2_floordiv(lltype.Signed, vv, wm1s) + q = vv // wm1 # can't use long2_floordiv(), q may not fit r = vv - wm1 * q while wm2 * q > ((r << SHIFT) | v.widedigit(abs(j-2))): q -= 1 From pypy.commits at gmail.com Sun Jun 26 16:51:43 2016 From: pypy.commits at gmail.com (arigo) Date: Sun, 26 Jun 2016 13:51:43 -0700 (PDT) Subject: [pypy-commit] pypy default: hg backout 800377eb1f02, 99ea9c77a44c, 507eee5b4548, 469aff993fa6 Message-ID: <5770405f.4a2e1c0a.889c3.ffff958b@mx.google.com> Author: Armin Rigo Branch: Changeset: r85391:84041f155fb8 Date: 2016-06-26 21:59 +0100 http://bitbucket.org/pypy/pypy/changeset/84041f155fb8/ Log: hg backout 800377eb1f02, 99ea9c77a44c, 507eee5b4548, 469aff993fa6 This plan didn't work out: the result is slower (on gcc linux64). I suspect now that it is because gcc normally turns "x / y", where y is a loop-constant, into a simpler instruction in the loop with precomputing something outside the loop. diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1827,8 +1827,6 @@ Divide bigint pin by non-zero digit n, storing quotient in pout, and returning the remainder. It's OK for pin == pout on entry. """ - from rpython.rtyper.lltypesystem.lloperation import llop - rem = _widen_digit(0) assert n > 0 and n <= MASK if not size: @@ -1836,9 +1834,9 @@ size -= 1 while size >= 0: rem = (rem << SHIFT) | pin.widedigit(size) - hi = llop.long2_floordiv(lltype.Signed, rem, n) + hi = rem // n pout.setdigit(size, hi) - rem -= _widen_digit(hi) * n + rem -= hi * n size -= 1 return rffi.cast(lltype.Signed, rem) @@ -1926,7 +1924,6 @@ z._normalize() return z _muladd1._annspecialcase_ = "specialize:argtype(2)" - def _v_lshift(z, a, m, d): """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put * result in z[0:m], and return the d bits shifted out of the top. @@ -2011,7 +2008,7 @@ vtop = v.widedigit(j) assert vtop <= wm1 vv = (vtop << SHIFT) | v.widedigit(abs(j-1)) - q = vv // wm1 # can't use long2_floordiv(), q may not fit + q = vv / wm1 r = vv - wm1 * q while wm2 * q > ((r << SHIFT) | v.widedigit(abs(j-2))): q -= 1 diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -319,9 +319,6 @@ 'lllong_rshift': LLOp(canfold=True), # args (r_longlonglong, int) 'lllong_xor': LLOp(canfold=True), - 'long2_floordiv': LLOp(canfold=True), # r = x / y: r and y Signed, - # x can have 2x the size - 'cast_primitive': LLOp(canfold=True), 'cast_bool_to_int': LLOp(canfold=True), 'cast_bool_to_uint': LLOp(canfold=True), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -16,7 +16,7 @@ 'bool': True, 'is_true':True} # global synonyms for some types -from rpython.rlib.rarithmetic import intmask, base_int +from rpython.rlib.rarithmetic import intmask from rpython.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong, r_longlonglong from rpython.rtyper.lltypesystem.llmemory import AddressAsInt @@ -733,16 +733,6 @@ assert isinstance(x, bool) return x -def op_long2_floordiv(x, y): - if lltype.typeOf(x) != lltype.Signed: - assert isinstance(x, base_int) - assert x.BITS == 2 * r_int.BITS - assert x.SIGNED - assert lltype.typeOf(y) is lltype.Signed - result = int(x) // y - assert result == intmask(result), "overflow in long2_floordiv" - return result - # ____________________________________________________________ def get_op_impl(opname): diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -106,13 +106,3 @@ #define PYPY_X86_CHECK_SSE2_DEFINED RPY_EXTERN void pypy_x86_check_sse2(void); #endif - - -#undef OP_LONG2_FLOORDIV -/* assumes that 'y' and 'r' fit in a signed word, - but 'x' takes up to two words */ -#define OP_LONG2_FLOORDIV(x, y, r) do { \ - long ignored; \ - __asm__("idiv %2" : "=a"(r), "=d"(ignored) : \ - "r"((long)y), "A"((long long)x)); \ - } while (0) diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -6,13 +6,3 @@ asm volatile("rdtsc" : "=a"(_rax), "=d"(_rdx)); \ val = (_rdx << 32) | _rax; \ } while (0) - -#undef OP_LONG2_FLOORDIV -/* assumes that 'y' and 'r' fit in a signed word, - but 'x' takes up to two words */ -#define OP_LONG2_FLOORDIV(x, y, r) do { \ - long ignored; \ - __asm__("idiv %2" : "=a"(r), "=d"(ignored) : \ - "r"((long)y), "a"((long)x), \ - "d"((long)((x >> 32) >> 32))); \ - } while (0) diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -135,7 +135,6 @@ #define OP_LLONG_FLOORDIV(x,y,r) r = (x) / (y) #define OP_ULLONG_FLOORDIV(x,y,r) r = (x) / (y) #define OP_LLLONG_FLOORDIV(x,y,r) r = (x) / (y) -#define OP_LONG2_FLOORDIV(x,y,r) r = (x) / (y) /* modulus */ diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py --- a/rpython/translator/c/test/test_lltyped.py +++ b/rpython/translator/c/test/test_lltyped.py @@ -1,7 +1,6 @@ -import py, sys, random +import py from rpython.rtyper.lltypesystem.lltype import * from rpython.rtyper.lltypesystem import rffi -from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator.c.test.test_genc import compile from rpython.tool.sourcetools import func_with_new_name @@ -1024,27 +1023,3 @@ assert fn(r_longlong(1)) == True assert fn(r_longlong(256)) == True assert fn(r_longlong(2**32)) == True - - def test_long2_floordiv(self): - def f(a, b): - return llop.long2_floordiv(Signed, a, b) - fn = self.getcompiled(f, [int, int]) - assert fn(100, 3) == 33 - # - if sys.maxint > 2**32: - HUGE = getattr(rffi, '__INT128_T', None) - bits = 128 - else: - HUGE = SignedLongLong - bits = 64 - if HUGE is not None: - def f(a, b, c): - ab = (rffi.cast(HUGE, a) << (bits//2)) | b - return llop.long2_floordiv(Signed, ab, c) - fn = self.getcompiled(f, [int, int, int]) - for i in range(100): - a = random.randrange(0, 10) - b = random.randrange(0, sys.maxint+1) - c = random.randrange(2*a+2, 25) - print a, b, c - assert fn(a, b, c) == ((a << (bits//2)) | b) // c From pypy.commits at gmail.com Mon Jun 27 04:02:00 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 01:02:00 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: Reference count fix Message-ID: <5770dd78.8159c20a.acef6.ffffcd8a@mx.google.com> Author: Armin Rigo Branch: PyTuple_Type-subclass Changeset: r85392:f5d3d1a77711 Date: 2016-06-27 10:03 +0200 http://bitbucket.org/pypy/pypy/changeset/f5d3d1a77711/ Log: Reference count fix diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -5,7 +5,7 @@ build_type_checkers, PyVarObjectFields, cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - make_ref, from_ref, decref, pyobj_has_w_obj, + make_ref, from_ref, decref, incref, pyobj_has_w_obj, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -83,7 +83,9 @@ except: while i > 0: i -= 1 - decref(space, py_tup.c_ob_item[i]) + ob = py_tup.c_ob_item[i] + py_tup.c_ob_item[i] = lltype.nullptr(PyObject.TO) + decref(space, ob) raise def tuple_realize(space, py_obj): @@ -98,9 +100,7 @@ p = py_tup.c_ob_item items_w = [None] * l for i in range(l): - w_item = None - if p[i]: - w_item = from_ref(space, p[i]) + w_item = from_ref(space, p[i]) if w_item is None: fatalerror_notb( "Fatal error in cpyext, CPython compatibility layer: " @@ -120,8 +120,7 @@ py_tup = rffi.cast(PyTupleObject, py_obj) p = py_tup.c_ob_item for i in range(py_tup.c_ob_size): - if p[i] and p[i].c_ob_refcnt > 0: - decref(space, p[i]) + decref(space, p[i]) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -195,7 +194,9 @@ else: to_cp = newsize for i in range(to_cp): - newref.c_ob_item[i] = oldref.c_ob_item[i] + ob = oldref.c_ob_item[i] + incref(space, ob) + newref.c_ob_item[i] = ob except: decref(space, p_ref[0]) p_ref[0] = lltype.nullptr(PyObject.TO) From pypy.commits at gmail.com Mon Jun 27 05:05:39 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 02:05:39 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Start to work on finalizers: split the log file into packets Message-ID: <5770ec63.c72d1c0a.52950.ffffa510@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85393:6e38ff9d8113 Date: 2016-06-27 11:06 +0200 http://bitbucket.org/pypy/pypy/changeset/6e38ff9d8113/ Log: Start to work on finalizers: split the log file into packets diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1,6 +1,7 @@ #include "common_header.h" #include #include +#include #include #include #include @@ -24,17 +25,19 @@ #define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) #define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) +//#define ASYNC_FINALIZER ((int16_t)0xff46) + typedef struct { Signed version; - Signed reserved1, reserved2; + uint64_t reserved1, reserved2; int argc; char **argv; } rdb_header_t; rpy_revdb_t rpy_revdb; -static char rpy_rev_buffer[16384]; +static char rpy_rev_buffer[16384]; /* max. 32768 */ static int rpy_rev_fileno = -1; static unsigned char flag_io_disabled; @@ -109,9 +112,6 @@ int i; assert(RPY_RDB_REPLAY == 0); - rpy_revdb.buf_p = rpy_rev_buffer; - rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; - rpy_revdb.unique_id_seen = 1; if (filename && *filename) { putenv("PYPYRDB="); @@ -137,20 +137,29 @@ h.argv = argv; write_all((const char *)&h, sizeof(h)); } + rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t); + rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; + rpy_revdb.unique_id_seen = 1; +} + +static void flush_buffer(void) +{ + /* write the current buffer content to the OS */ + ssize_t size = rpy_revdb.buf_p - rpy_rev_buffer; + rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t); + if (rpy_rev_fileno >= 0) + write_all(rpy_rev_buffer, size); } RPY_EXTERN void rpy_reverse_db_flush(void) { - /* write the current buffer content to the OS */ - - ssize_t wsize, size = rpy_revdb.buf_p - rpy_rev_buffer; - char *p; - rpy_revdb.buf_p = rpy_rev_buffer; - if (size == 0 || rpy_rev_fileno < 0) - return; - - write_all(rpy_rev_buffer, size); + ssize_t packet_size = rpy_revdb.buf_p - (rpy_rev_buffer + sizeof(int16_t)); + if (packet_size != 0) { + assert(0 < packet_size && packet_size <= 32767); + *(int16_t *)rpy_rev_buffer = packet_size; + flush_buffer(); + } } RPY_EXTERN @@ -176,7 +185,7 @@ return obj->h_hash; } -static int64_t recording_offset(void) +static uint64_t recording_offset(void) { off_t base_offset; ssize_t extra_size = rpy_revdb.buf_p - rpy_rev_buffer; @@ -516,6 +525,7 @@ rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; + rpy_revdb.buf_readend = rpy_rev_buffer; rpy_revdb.stop_point_break = 1; rpy_revdb.unique_id_seen = 1; @@ -528,19 +538,44 @@ signal(SIGCHLD, SIG_IGN); } +static void fetch_more(ssize_t keep, ssize_t expected_size) +{ + ssize_t rsize; + if (rpy_revdb.buf_p != rpy_rev_buffer) + memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep); + rsize = read_at_least(rpy_rev_buffer + keep, + expected_size - keep, + sizeof(rpy_rev_buffer) - keep); + rpy_revdb.buf_p = rpy_rev_buffer; + rpy_revdb.buf_readend = rpy_rev_buffer + keep + rsize; + /* rpy_revdb.buf_limit is not set */ +} + RPY_EXTERN -char *rpy_reverse_db_fetch(int expected_size, const char *file, int line) +char *rpy_reverse_db_fetch(const char *file, int line) { if (!flag_io_disabled) { - ssize_t rsize, keep = rpy_revdb.buf_limit - rpy_revdb.buf_p; + ssize_t keep; + ssize_t full_packet_size; + if (rpy_revdb.buf_limit != rpy_revdb.buf_p) { + fprintf(stderr, "bad log format: incomplete packet\n"); + exit(1); + } + keep = rpy_revdb.buf_readend - rpy_revdb.buf_p; assert(keep >= 0); - memmove(rpy_rev_buffer, rpy_revdb.buf_p, keep); - rsize = read_at_least(rpy_rev_buffer + keep, - expected_size - keep, - sizeof(rpy_rev_buffer) - keep); - rpy_revdb.buf_p = rpy_rev_buffer; - rpy_revdb.buf_limit = rpy_rev_buffer + keep + rsize; - return rpy_rev_buffer; + + if (keep < sizeof(int16_t)) { + /* 'keep' does not even contain the next packet header */ + fetch_more(keep, sizeof(int16_t)); + keep = rpy_revdb.buf_readend - rpy_rev_buffer; + } + full_packet_size = sizeof(int16_t) + *(int16_t *)rpy_revdb.buf_p; + if (keep < full_packet_size) { + fetch_more(keep, full_packet_size); + } + rpy_revdb.buf_limit = rpy_revdb.buf_p + full_packet_size; + rpy_revdb.buf_p += sizeof(int16_t); + return rpy_revdb.buf_p; } else { /* this is called when we are in execute_rpy_command(): we are diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -16,7 +16,7 @@ #define RPY_RDB_DYNAMIC_REPLAY #endif bool_t watch_enabled; - char *buf_p, *buf_limit; + char *buf_p, *buf_limit, *buf_readend; uint64_t stop_point_seen, stop_point_break; uint64_t unique_id_seen, unique_id_break; } rpy_revdb_t; @@ -53,8 +53,7 @@ char *_src = rpy_revdb.buf_p; \ char *_end1 = _src + sizeof(_e); \ if (_end1 > rpy_revdb.buf_limit) { \ - _src = rpy_reverse_db_fetch(sizeof(_e), \ - __FILE__, __LINE__); \ + _src = rpy_reverse_db_fetch(__FILE__, __LINE__); \ _end1 = _src + sizeof(_e); \ } \ rpy_revdb.buf_p = _end1; \ @@ -114,8 +113,7 @@ r = rpy_reverse_db_weakref_deref(weakref) RPY_EXTERN void rpy_reverse_db_flush(void); -RPY_EXTERN char *rpy_reverse_db_fetch(int expected_size, - const char *file, int line); +RPY_EXTERN char *rpy_reverse_db_fetch(const char *file, int line); RPY_EXTERN void rpy_reverse_db_stop_point(void); RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2, int64_t arg3, RPyString *extra); diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -21,19 +21,29 @@ assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n' # self.cur = 0 - x = self.next('c'); assert x == '\x00' - x = self.next(); assert x == 0x00FF0001 - x = self.next(); assert x == 0 - x = self.next(); assert x == 0 - self.argc = self.next() - self.argv = self.next() + x = self.read1('c'); assert x == '\x00' + x = self.read1('P'); assert x == 0x00FF0001 + x = self.read1('P'); assert x == 0 + x = self.read1('P'); assert x == 0 + self.argc = self.read1('P') + self.argv = self.read1('P') + self.current_packet_end = self.cur self.read_check_argv(expected_argv) - def next(self, mode='P'): + def read1(self, mode): p = self.cur self.cur = p + struct.calcsize(mode) return struct.unpack_from(mode, self.buffer, p)[0] + def next(self, mode='P'): + if self.current_packet_end == self.cur: + packet_size = self.read1('h') + assert packet_size > 0 + self.current_packet_end = self.cur + packet_size + result = self.read1(mode) + assert self.cur <= self.current_packet_end + return result + def read_check_argv(self, expected): assert self.argc == len(expected) for i in range(self.argc): diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -1,5 +1,5 @@ -import weakref, gc -from rpython.rlib import revdb +import weakref +from rpython.rlib import revdb, rgc from rpython.rlib.debug import debug_print from rpython.rlib.objectmodel import keepalive_until_here from rpython.translator.revdb.message import * @@ -23,7 +23,7 @@ WEAKREF_AFTERWARDS_ALIVE = chr(0xeb) -class TestRecordingWeakref(BaseRecordingTests): +class TestRecording(BaseRecordingTests): def test_weakref_create(self): class X: @@ -71,8 +71,25 @@ x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() + def test_finalizer_light_ignored(self): + class X: + @rgc.must_be_light_finalizer + def __del__(self): + pass + def main(argv): + lst = [X() for i in range(3000)] + for i in range(3000): + lst[i] = None + revdb.stop_point() + return 9 + self.compile(main, [], backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + x = rdb.next('q'); assert x == 3000 # number of stop points + assert rdb.done() -class TestReplayingWeakref(InteractiveTests): + +class TestReplaying(InteractiveTests): expected_stop_points = 1 def setup_class(cls): @@ -108,7 +125,7 @@ print ''.join(outp) if (j % 1000) == 999: debug_print('============= COLLECT ===========') - gc.collect() + rgc.collect() debug_print('------ done', j, '.') assert not dead[0] assert not dead[-1] From pypy.commits at gmail.com Mon Jun 27 05:07:09 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 02:07:09 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: hg merge default Message-ID: <5770ecbd.e40bc30a.3361f.4fd0@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85394:b32e244515cd Date: 2016-06-27 11:07 +0200 http://bitbucket.org/pypy/pypy/changeset/b32e244515cd/ Log: hg merge default diff too long, truncating to 2000 out of 3264 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -26,3 +26,4 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 +7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.9 +Version: 0.4.10 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.9" +__version__ = "0.4.10" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -315,13 +315,28 @@ - ``complex`` + - ``str`` (empty or single-character strings only) + + - ``unicode`` (empty or single-character strings only) + + - ``tuple`` (empty tuples only) + + - ``frozenset`` (empty frozenset only) + This change requires some changes to ``id`` as well. ``id`` fulfills the following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the above types will return a value that is computed from the argument, and can thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long). -Notably missing from the list above are ``str`` and ``unicode``. If your -code relies on comparing strings with ``is``, then it might break in PyPy. +Note that strings of length 2 or greater can be equal without being +identical. Similarly, ``x is (2,)`` is not necessarily true even if +``x`` contains a tuple and ``x == (2,)``. The uniqueness rules apply +only to the particular cases described above. The ``str``, ``unicode``, +``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a +test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was +equal to ``"?"`` or ``()``. The new behavior added in PyPy 5.4 is +closer to CPython's, which caches precisely the empty tuple/frozenset, +and (generally but not always) the strings and unicodes of length <= 1. Note that for floats there "``is``" only one object per "bit pattern" of the float. So ``float('nan') is float('nan')`` is true on PyPy, diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.1.rst whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.1.rst @@ -0,0 +1,41 @@ +========== +PyPy 5.3.1 +========== + +We have released a bugfix for PyPy2.7-v5.3.0, released last week, +due to issues_ reported by users. + +Thanks to those who reported the issues. + +.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,10 +1,14 @@ -========================= +========================== What's new in PyPy2.7 5.3+ -========================= +========================== .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 +.. 418b05f95db5 +Improve CPython compatibility for ``is``. Now code like ``if x is ():`` +works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id . + .. pull request #455 Add sys.{get,set}dlopenflags, for cpyext extensions. @@ -31,3 +35,16 @@ Simplify handling of interp-level tests and make it more forward- compatible. +.. branch: pyfile-tell +Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile + +.. branch: rw-PyString_AS_STRING +Allow rw access to the char* returned from PyString_AS_STRING, also refactor +PyStringObject to look like cpython's and allow subclassing PyString_Type and +PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst @@ -0,0 +1,15 @@ +=========================== +What's new in PyPy2.7 5.3.1 +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.3.0 +.. startrev: f4d726d1a010 + + +A bug-fix release, merging these changes: + + * Add include guards to pymem.h, fixes issue #2321 + + * Make vmprof build on OpenBSD, from pull request #456 + + * Fix ``bytearray('').replace('a', 'ab')``, issue #2324 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -800,6 +800,21 @@ pypy_debug_catch_fatal_exception() assert False +def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid): + from rpython.rlib import rgil + # see "Handling of the GIL" above + assert cpyext_glob_tid_ptr[0] == 0 + if pygilstate_release: + from pypy.module.cpyext import pystate + unlock = (gilstate == pystate.PyGILState_UNLOCKED) + else: + unlock = gil_release or _gil_auto + if unlock: + rgil.release() + else: + cpyext_glob_tid_ptr[0] = tid + + def make_wrapper_second_level(space, argtypesw, restype, result_kind, error_value, gil): from rpython.rlib import rgil @@ -827,6 +842,7 @@ def wrapper_second_level(callable, pname, *args): from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj from pypy.module.cpyext.pyobject import as_pyobj + from pypy.module.cpyext import pystate # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -839,7 +855,6 @@ rgil.acquire() assert cpyext_glob_tid_ptr[0] == 0 elif pygilstate_ensure: - from pypy.module.cpyext import pystate if cpyext_glob_tid_ptr[0] == tid: cpyext_glob_tid_ptr[0] = 0 args += (pystate.PyGILState_LOCKED,) @@ -850,6 +865,10 @@ if cpyext_glob_tid_ptr[0] != tid: no_gil_error(pname) cpyext_glob_tid_ptr[0] = 0 + if pygilstate_release: + gilstate = rffi.cast(lltype.Signed, args[-1]) + else: + gilstate = pystate.PyGILState_IGNORE rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -919,24 +938,13 @@ except Exception as e: unexpected_exception(pname, e, tb) + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return fatal_value assert lltype.typeOf(retval) == restype rffi.stackcounter.stacks_counter -= 1 - # see "Handling of the GIL" above - assert cpyext_glob_tid_ptr[0] == 0 - if pygilstate_release: - from pypy.module.cpyext import pystate - arg = rffi.cast(lltype.Signed, args[-1]) - unlock = (arg == pystate.PyGILState_UNLOCKED) - else: - unlock = gil_release or _gil_auto - if unlock: - rgil.release() - else: - cpyext_glob_tid_ptr[0] = tid - + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return retval wrapper_second_level._dont_inline_ = True @@ -1202,8 +1210,6 @@ cpyext_type_init = self.cpyext_type_init self.cpyext_type_init = None for pto, w_type in cpyext_type_init: - if space.is_w(w_type, space.w_str): - pto.c_tp_itemsize = 1 finish_type_1(space, pto) finish_type_2(space, pto, w_type) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -6,7 +6,9 @@ from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, - make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref) + make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref, + pyobj_has_w_obj) +from pypy.objspace.std.bytesobject import W_BytesObject ## ## Implementation of PyStringObject @@ -16,7 +18,7 @@ ## ----------- ## ## PyString_AsString() must return a (non-movable) pointer to the underlying -## buffer, whereas pypy strings are movable. C code may temporarily store +## ob_sval, whereas pypy strings are movable. C code may temporarily store ## this address and use it, as long as it owns a reference to the PyObject. ## There is no "release" function to specify that the pointer is not needed ## any more. @@ -28,7 +30,7 @@ ## -------- ## ## PyStringObject contains two additional members: the ob_size and a pointer to a -## char buffer; it may be NULL. +## char ob_sval; it may be NULL. ## ## - A string allocated by pypy will be converted into a PyStringObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is @@ -41,6 +43,9 @@ ## the pypy string is created, and added to the global map of tracked ## objects. The buffer is then supposed to be immutable. ## +##- A buffer obtained from PyString_AS_STRING() could be mutable iff +## there is no corresponding pypy object for the string +## ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a ## similar object. ## @@ -53,7 +58,7 @@ PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) PyStringObjectFields = PyVarObjectFields + \ - (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP)) + (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) @bootstrap_function @@ -69,44 +74,43 @@ def new_empty_str(space, length): """ - Allocate a PyStringObject and its buffer, but without a corresponding - interpreter object. The buffer may be mutated, until string_realize() is + Allocate a PyStringObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until string_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) - py_obj = typedescr.allocate(space, space.w_str) + py_obj = typedescr.allocate(space, space.w_str, length) py_str = rffi.cast(PyStringObject, py_obj) - - buflen = length + 1 - py_str.c_ob_size = length - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, - flavor='raw', zero=True, - add_memory_pressure=True) + py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str def string_attach(space, py_obj, w_obj): """ - Fills a newly allocated PyStringObject with the given string object. The - buffer must not be modified. + Copy RPython string object contents to a PyStringObject. The + c_ob_sval must not be modified. """ py_str = rffi.cast(PyStringObject, py_obj) - py_str.c_ob_size = len(space.str_w(w_obj)) - py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + s = space.str_w(w_obj) + if py_str.c_ob_size < len(s): + raise oefmt(space.w_ValueError, + "string_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) + rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) + py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL def string_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject buffer must not + Creates the string in the interpreter. The PyStringObject ob_sval must not be modified after this call. """ py_str = rffi.cast(PyStringObject, py_obj) - if not py_str.c_buffer: - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1, - flavor='raw', zero=True) - s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size) - w_obj = space.wrap(s) + s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_BytesObject, w_type) + w_obj.__init__(s) py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL track_reference(space, py_obj, w_obj) @@ -116,9 +120,6 @@ def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ - py_str = rffi.cast(PyStringObject, py_obj) - if py_str.c_buffer: - lltype.free(py_str.c_buffer, flavor="raw") from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) @@ -139,6 +140,9 @@ @cpython_api([PyObject], rffi.CCHARP, error=0) def PyString_AsString(space, ref): + return _PyString_AsString(space, ref) + +def _PyString_AsString(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: pass # typecheck returned "ok" without forcing 'ref' at all elif not PyString_Check(space, ref): # otherwise, use the alternate way @@ -151,15 +155,23 @@ "expected string or Unicode object, %T found", from_ref(space, ref)) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - return ref_str.c_buffer + if not pyobj_has_w_obj(ref): + # XXX Force the ref? + string_realize(space, ref) + return ref_str.c_ob_sval + + at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) +def PyString_AS_STRING(space, void_ref): + ref = rffi.cast(PyObject, void_ref) + # if no w_str is associated with this ref, + # return the c-level ptr as RW + if not pyobj_has_w_obj(ref): + py_str = rffi.cast(PyStringObject, ref) + return py_str.c_ob_sval + return _PyString_AsString(space, ref) @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) -def PyString_AsStringAndSize(space, ref, buffer, length): +def PyString_AsStringAndSize(space, ref, data, length): if not PyString_Check(space, ref): from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) @@ -169,18 +181,16 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) + if not pyobj_has_w_obj(ref): + # force the ref + string_realize(space, ref) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - buffer[0] = ref_str.c_buffer + data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size else: i = 0 - while ref_str.c_buffer[i] != '\0': + while ref_str.c_ob_sval[i] != '\0': i += 1 if i != ref_str.c_ob_size: raise oefmt(space.w_TypeError, @@ -209,10 +219,10 @@ set to NULL, a memory exception is set, and -1 is returned. """ # XXX always create a new string so far - py_str = rffi.cast(PyStringObject, ref[0]) - if not py_str.c_buffer: + if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") + py_str = rffi.cast(PyStringObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: @@ -224,7 +234,7 @@ if oldsize < newsize: to_cp = oldsize for i in range(to_cp): - py_newstr.c_buffer[i] = py_str.c_buffer[i] + py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i] Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.3.1-alpha0" -#define PYPY_VERSION_NUM 0x05030100 +#define PYPY_VERSION "5.3.2-alpha0" +#define PYPY_VERSION_NUM 0x05030200 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h --- a/pypy/module/cpyext/include/stringobject.h +++ b/pypy/module/cpyext/include/stringobject.h @@ -10,7 +10,6 @@ #include #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op)) -#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op)) /* Type PyStringObject represents a character string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is @@ -41,12 +40,11 @@ PyObject_VAR_HEAD long ob_shash; int ob_sstate; - char * buffer; /* change the name from cpython so all non-api c access is thwarted */ + char ob_sval[1]; /* Invariants - * (not relevant in PyPy, all stringobjects are backed by a pypy object) - * buffer contains space for 'ob_size+1' elements. - * buffer[ob_size] == 0. + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. * ob_shash is the hash of the string or -1 if not computed yet. * ob_sstate != 0 iff the string object is in stringobject.c's * 'interned' dictionary; in this case the two references diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -55,6 +55,7 @@ if not PyFile_Check(space, w_p): raise oefmt(space.w_IOError, 'first argument must be an open file') assert isinstance(w_p, W_File) + w_p.stream.flush_buffers() try: fd = space.int_w(space.call_method(w_p, 'fileno')) mode = w_p.mode diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, - INTERPLEVEL_API) + INTERPLEVEL_API, PyVarObject) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject @@ -47,13 +47,16 @@ size = pytype.c_tp_basicsize else: size = rffi.sizeof(self.basestruct) - if itemcount and w_type is not space.w_str: + if pytype.c_tp_itemsize: size += itemcount * pytype.c_tp_itemsize assert size >= rffi.sizeof(PyObject.TO) buf = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True, add_memory_pressure=True) pyobj = rffi.cast(PyObject, buf) + if pytype.c_tp_itemsize: + pyvarobj = rffi.cast(PyVarObject, pyobj) + pyvarobj.c_ob_size = itemcount pyobj.c_ob_refcnt = 1 #pyobj.c_ob_pypy_link should get assigned very quickly pyobj.c_ob_type = pytype @@ -152,13 +155,18 @@ class InvalidPointerException(Exception): pass -def create_ref(space, w_obj, itemcount=0): +def create_ref(space, w_obj): """ Allocates a PyObject, and fills its fields with info from the given interpreter object. """ w_type = space.type(w_obj) + pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) + if pytype.c_tp_itemsize != 0: + itemcount = space.len_w(w_obj) # PyStringObject and subclasses + else: + itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) track_reference(space, py_obj, w_obj) # diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -207,6 +207,7 @@ PyGILState_STATE = rffi.INT PyGILState_LOCKED = 0 PyGILState_UNLOCKED = 1 +PyGILState_IGNORE = 2 ExecutionContext.cpyext_gilstate_counter_noleave = 0 diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c --- a/pypy/module/cpyext/src/stringobject.c +++ b/pypy/module/cpyext/src/stringobject.c @@ -107,7 +107,7 @@ if (!string) return NULL; - s = PyString_AsString(string); + s = PyString_AS_STRING(string); for (f = format; *f; f++) { if (*f == '%') { diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,4 +1,4 @@ -import py +import os import pytest def pytest_configure(config): @@ -21,3 +21,14 @@ def pytest_funcarg__api(request): return request.cls.api +if os.name == 'nt': + @pytest.yield_fixture(autouse=True, scope='session') + def prevent_dialog_box(): + """Do not open dreaded dialog box on segfault on Windows""" + import ctypes + SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN + old_err_mode = ctypes.windll.kernel32.GetErrorMode() + new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX + ctypes.windll.kernel32.SetErrorMode(new_err_mode) + yield + ctypes.windll.kernel32.SetErrorMode(old_err_mode) diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -43,15 +43,6 @@ /* foo methods */ -static void -foo_dealloc(fooobject *foop) -{ - PyObject_Del(foop); -} - - -/* foo methods-as-attributes */ - static PyObject * foo_copy(fooobject *self) { @@ -195,7 +186,7 @@ sizeof(fooobject), /*tp_size*/ 0, /*tp_itemsize*/ /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -463,9 +454,9 @@ PyTypeObject InitErrType = { PyObject_HEAD_INIT(NULL) 0, - "foo.InitErr", - sizeof(PyObject), - 0, + "foo.InitErrType", + sizeof(PyObject),/*tp_basicsize*/ + 0, /*tp_itemsize*/ 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ @@ -508,12 +499,12 @@ 0, /*tp_dictoffset*/ initerrtype_init, /*tp_init*/ - 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_alloc*/ 0, /*tp_new*/ - 0, /*tp_free Low-level free-memory routine */ - 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ 0, /*tp_bases*/ - 0, /*tp_mro method resolution order */ + 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0 /*tp_weaklist*/ diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/support.py @@ -0,0 +1,68 @@ +import os +import py +from sys import platform + +if os.name != 'nt': + so_ext = 'so' +else: + so_ext = 'dll' + +def c_compile(cfilenames, outputfilename, + compile_extra=None, link_extra=None, + include_dirs=None, libraries=None, library_dirs=None): + compile_extra = compile_extra or [] + link_extra = link_extra or [] + include_dirs = include_dirs or [] + libraries = libraries or [] + library_dirs = library_dirs or [] + if platform == 'win32': + link_extra = link_extra + ['/DEBUG'] # generate .pdb file + if platform == 'darwin': + # support Fink & Darwinports + for s in ('/sw/', '/opt/local/'): + if (s + 'include' not in include_dirs + and os.path.exists(s + 'include')): + include_dirs.append(s + 'include') + if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): + library_dirs.append(s + 'lib') + + outputfilename = py.path.local(outputfilename).new(ext=so_ext) + saved_environ = os.environ.copy() + try: + _build( + cfilenames, outputfilename, + compile_extra, link_extra, + include_dirs, libraries, library_dirs) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(cfilenames, outputfilename, compile_extra, link_extra, + include_dirs, libraries, library_dirs): + from distutils.ccompiler import new_compiler + from distutils import sysconfig + compiler = new_compiler(force=1) + sysconfig.customize_compiler(compiler) # XXX + objects = [] + for cfile in cfilenames: + cfile = py.path.local(cfile) + old = cfile.dirpath().chdir() + try: + res = compiler.compile([cfile.basename], + include_dirs=include_dirs, extra_preargs=compile_extra) + assert len(res) == 1 + cobjfile = py.path.local(res[0]) + assert cobjfile.check() + objects.append(str(cobjfile)) + finally: + old.chdir() + + compiler.link_shared_object( + objects, str(outputfilename), + libraries=libraries, + extra_preargs=link_extra, + library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestStringObject(AppTestCpythonExtensionBase): def test_basic(self): module = self.import_extension('foo', [ @@ -16,24 +17,10 @@ """ PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); int result = 0; - size_t expected_size; if(PyByteArray_Size(s) == 12) { result = 1; } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*3; - #elif defined Py_DEBUG - expected_size = 64; - #else - expected_size = 48; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), @@ -53,7 +40,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyByteArray_FromStringAndSize(NULL, 4); if (s == NULL) @@ -84,11 +70,10 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); - memcpy(PyByteArray_AS_STRING(base), "works", 6); + memcpy(PyByteArray_AS_STRING(base), "works", 6); Py_INCREF(base); return base; """), @@ -115,6 +100,7 @@ assert s == 'test' def test_manipulations(self): + import sys module = self.import_extension('foo', [ ("bytearray_from_string", "METH_VARARGS", ''' @@ -141,9 +127,9 @@ ("concat", "METH_VARARGS", """ PyObject * ret, *right, *left; - PyObject *ba1, *ba2; + PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -157,7 +143,9 @@ """)]) assert module.bytearray_from_string("huheduwe") == "huhe" assert module.str_from_bytearray(bytearray('abc')) == 'abc' - raises(ValueError, module.str_from_bytearray, 4.0) + if '__pypy__' in sys.builtin_module_names: + # CPython only makes an assert. + raises(ValueError, module.str_from_bytearray, 4.0) ret = module.concat('abc', 'def') assert ret == 'abcdef' assert not isinstance(ret, str) @@ -171,9 +159,9 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } - + ba = PyByteArray_FromObject(obj); if (ba == NULL) return NULL; @@ -187,7 +175,7 @@ { printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); return NULL; - } + } return ba; ''' )]) diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,31 +25,15 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyString_FromString("Hello world"); - int result = 0; - size_t expected_size; + int result = PyString_Size(s); - if(PyString_Size(s) == 11) { - result = 1; - } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*7; - #elif defined Py_DEBUG - expected_size = 53; - #else - expected_size = 37; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); - return PyBool_FromLong(result); + return PyLong_FromLong(result); """), ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyString_Size(f); + PyString_Size(f); Py_DECREF(f); return NULL; @@ -60,7 +44,7 @@ """)], prologue='#include ') assert module.get_hello1() == 'Hello world' assert module.get_hello2() == 'Hello world' - assert module.test_Size() + assert module.test_Size() == 11 raises(TypeError, module.test_Size_exception) assert module.test_is_string("") @@ -72,7 +56,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyString_FromStringAndSize(NULL, 4); if (s == NULL) @@ -81,7 +64,7 @@ if (t == NULL) return NULL; Py_DECREF(t); - c = PyString_AsString(s); + c = PyString_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -100,7 +83,6 @@ PyObject *base; PyTypeObject * type; PyStringObject *obj; - char * p_str; base = PyString_FromString("test"); if (PyString_GET_SIZE(base) != 4) return PyLong_FromLong(-PyString_GET_SIZE(base)); @@ -110,14 +92,22 @@ obj = (PyStringObject*)type->tp_alloc(type, 10); if (PyString_GET_SIZE(obj) != 10) return PyLong_FromLong(PyString_GET_SIZE(obj)); - /* cannot work, there is only RO access - memcpy(PyString_AS_STRING(obj), "works", 6); */ + /* cannot work, there is only RO access */ + memcpy(PyString_AS_STRING(obj), "works", 6); Py_INCREF(obj); return (PyObject*)obj; """), + ('alloc_rw', "METH_NOARGS", + ''' + PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); + memcpy(PyString_AS_STRING(obj), "works", 6); + return (PyObject*)obj; + '''), ]) + s = module.alloc_rw() + assert s == 'works' + '\x00' * 5 s = module.tpalloc() - assert s == '\x00' * 10 + assert s == 'works' + '\x00' * 5 def test_AsString(self): module = self.import_extension('foo', [ @@ -312,17 +302,17 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyStringObject*)obj)->ob_shash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - int sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*int sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); '''), @@ -332,27 +322,124 @@ # doesn't really test, but if printf is enabled will prove sstate assert module.test_sstate() + def test_subclass(self): + # taken from PyStringArrType_Type in numpy's scalartypes.c.src + module = self.import_extension('bar', [ + ("newsubstr", "METH_O", + """ + PyObject * obj; + char * data; + int len; + PyType_Ready(&PyStringArrType_Type); + + data = PyString_AS_STRING(args); + len = PyString_GET_SIZE(args); + if (data == NULL || len < 1) + Py_RETURN_NONE; + obj = PyArray_Scalar(data, len); + return obj; + """), + ], prologue=""" + #include + PyTypeObject PyStringArrType_Type = { + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ + "bar.string_", /* tp_name*/ + sizeof(PyStringObject), /* tp_basicsize*/ + 0 /* tp_itemsize */ + }; + + static PyObject * + stringtype_repr(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + static PyObject * + stringtype_str(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + PyObject * + PyArray_Scalar(char *data, int n) + { + PyTypeObject *type = &PyStringArrType_Type; + PyObject *obj; + void *destptr; + int itemsize = n; + obj = type->tp_alloc(type, itemsize); + if (obj == NULL) { + return NULL; + } + destptr = PyString_AS_STRING(obj); + ((PyStringObject *)obj)->ob_shash = -1; + memcpy(destptr, data, itemsize); + return obj; + } + """, more_init = ''' + PyStringArrType_Type.tp_alloc = NULL; + PyStringArrType_Type.tp_free = NULL; + + PyStringArrType_Type.tp_repr = stringtype_repr; + PyStringArrType_Type.tp_str = stringtype_str; + PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE; + PyStringArrType_Type.tp_itemsize = sizeof(char); + PyStringArrType_Type.tp_base = &PyString_Type; + ''') + + a = module.newsubstr('abc') + assert type(a).__name__ == 'string_' + assert a == 'abc' class TestString(BaseApiTest): def test_string_resize(self, space, api): py_str = new_empty_str(space, 10) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - py_str.c_buffer[0] = 'a' - py_str.c_buffer[1] = 'b' - py_str.c_buffer[2] = 'c' + py_str.c_ob_sval[0] = 'a' + py_str.c_ob_sval[1] = 'b' + py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 3) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 3 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[3] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 10) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 10 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[10] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[10] == '\x00' Py_DecRef(space, ar[0]) lltype.free(ar, flavor='raw') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -7,8 +7,6 @@ from pypy import pypydir from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator import platform from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir from pypy.module.cpyext import api @@ -18,20 +16,7 @@ from rpython.tool import leakfinder from rpython.rlib import rawrefcount -def setup_module(module): - if os.name == 'nt': - # Do not open dreaded dialog box on segfault - import ctypes - SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN - old_err_mode = ctypes.windll.kernel32.GetErrorMode() - new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX - ctypes.windll.kernel32.SetErrorMode(new_err_mode) - module.old_err_mode = old_err_mode - -def teardown_module(module): - if os.name == 'nt': - import ctypes - ctypes.windll.kernel32.SetErrorMode(module.old_err_mode) +from .support import c_compile @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -46,7 +31,30 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def compile_extension_module(space, modname, include_dirs=[], **kwds): +def convert_sources_to_files(sources, dirname): + files = [] + for i, source in enumerate(sources): + filename = dirname / ('source_%d.c' % i) + with filename.open('w') as f: + f.write(str(source)) + files.append(filename) + return files + +def create_so(modname, include_dirs, source_strings=None, source_files=None, + compile_extra=None, link_extra=None, libraries=None): + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=compile_extra, link_extra=link_extra, + include_dirs=include_dirs, + libraries=libraries) + return soname + +def compile_extension_module(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -60,38 +68,36 @@ state = space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': - kwds["libraries"] = [api_library] + libraries = [api_library] # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] + compile_extra = ["/we4013"] # prevent linking with PythonXX.lib w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] - kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % (space.int_w(w_maj), space.int_w(w_min))] - elif sys.platform == 'darwin': - kwds["link_files"] = [str(api_library + '.dylib')] else: - kwds["link_files"] = [str(api_library + '.so')] + libraries = [] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror", "-g", "-O0"] - kwds["link_extra"]=["-g"] + compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + link_extra = ["-g"] + else: + compile_extra = link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs=api.include_dirs + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=api.include_dirs + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra, + libraries=libraries) from pypy.module.imp.importing import get_so_extension pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) soname.rename(pydname) return str(pydname) -def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds): +def compile_extension_module_applevel(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -103,24 +109,23 @@ build the module (so specify your source with one of those). """ if sys.platform == 'win32': - kwds["compile_extra"] = ["/we4013"] - kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] + compile_extra = ["/we4013"] + link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] elif sys.platform == 'darwin': + compile_extra = link_extra = None pass elif sys.platform.startswith('linux'): - kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"] + compile_extra = [ + "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] + link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs = [space.include_dir] + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=[space.include_dir] + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra) return str(soname) def freeze_refcnts(self): @@ -285,8 +290,8 @@ separate_module_sources = [] pydname = self.compile_extension_module( space, name, - separate_module_files=separate_module_files, - separate_module_sources=separate_module_sources) + source_files=separate_module_files, + source_strings=separate_module_sources) return space.wrap(pydname) @gateway.unwrap_spec(name=str, init='str_or_None', body=str, @@ -315,6 +320,11 @@ /* fix for cpython 2.7 Python.h if running tests with -A since pypy compiles with -fvisibility-hidden */ #undef PyMODINIT_FUNC + #ifdef __GNUC__ + # define RPY_EXPORTED extern __attribute__((visibility("default"))) + #else + # define RPY_EXPORTED extern __declspec(dllexport) + #endif #define PyMODINIT_FUNC RPY_EXPORTED void %(body)s @@ -326,16 +336,16 @@ """ % dict(name=name, init=init, body=body, PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' if PY_SSIZE_T_CLEAN else '') - kwds = dict(separate_module_sources=[code]) + kwds = dict(source_strings=[code]) else: assert not PY_SSIZE_T_CLEAN if filename is None: filename = name filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(separate_module_files=[filename]) - kwds['include_dirs'] = include_dirs - mod = self.compile_extension_module(space, name, **kwds) + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module(space, name, + include_dirs=include_dirs, **kwds) if load_it: if self.runappdirect: @@ -975,7 +985,7 @@ ('bar', 'METH_NOARGS', ''' /* reuse a name that is #defined in structmember.h */ - int RO; + int RO = 0; (void)RO; Py_RETURN_NONE; ''' ), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -142,7 +142,7 @@ 2000, 6, 6, 6, 6, 6, 6, Py_None, PyDateTimeAPI->DateTimeType); """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.new_date() == datetime.date(2000, 6, 6) assert module.new_time() == datetime.time(6, 6, 6, 6) @@ -241,6 +241,9 @@ PyObject* obj = PyDelta_FromDSU(6, 6, 6); PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 + // These macros are only defined in CPython 3.x and PyPy. + // See: http://bugs.python.org/issue13727 PyDateTime_DELTA_GET_DAYS(obj); PyDateTime_DELTA_GET_DAYS(delta); @@ -249,10 +252,10 @@ PyDateTime_DELTA_GET_MICROSECONDS(obj); PyDateTime_DELTA_GET_MICROSECONDS(delta); - +#endif return obj; """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.test_date_macros() == datetime.date(2000, 6, 6) assert module.test_datetime_macros() == datetime.datetime( diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py --- a/pypy/module/cpyext/test/test_frameobject.py +++ b/pypy/module/cpyext/test/test_frameobject.py @@ -13,7 +13,7 @@ PyObject *empty_string = PyString_FromString(""); PyObject *empty_tuple = PyTuple_New(0); PyCodeObject *py_code; - PyFrameObject *py_frame; + PyFrameObject *py_frame = NULL; py_code = PyCode_New( 0, /*int argcount,*/ @@ -75,7 +75,7 @@ """ int check; PyObject *type, *value, *tb; - PyObject *ret = PyRun_String("XXX", Py_eval_input, + PyObject *ret = PyRun_String("XXX", Py_eval_input, Py_None, Py_None); if (ret) { Py_DECREF(ret); diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -149,7 +149,6 @@ pybuffer = self.import_parser( ''' Py_buffer buf1, buf2, buf3; - PyObject *result; if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) { return NULL; } diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.micronumpy.ndarray import W_NDimArray from pypy.module.micronumpy.descriptor import get_dtype_cache -import pypy.module.micronumpy.constants as NPY +import pypy.module.micronumpy.constants as NPY def scalar(space): dtype = get_dtype_cache(space).w_float64dtype @@ -237,7 +237,7 @@ except: skip('numpy not importable') else: - numpy_incl = os.path.abspath(os.path.dirname(__file__) + + numpy_incl = os.path.abspath(os.path.dirname(__file__) + '/../include/_numpypy') assert os.path.exists(numpy_incl) cls.w_numpy_include = cls.space.wrap([numpy_incl]) @@ -273,7 +273,7 @@ { /* Should have failed */ Py_DECREF(obj1); - return NULL; + return NULL; } return obj1; ''' @@ -300,14 +300,14 @@ ), ("test_DescrFromType", "METH_O", """ - Signed typenum = PyInt_AsLong(args); + long typenum = PyInt_AsLong(args); return PyArray_DescrFromType(typenum); """ ), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -315,7 +315,7 @@ #define PyArray_FromObject _PyArray_FromObject #define PyArray_FromAny _PyArray_FromAny #endif - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -349,14 +349,14 @@ Py_INCREF(obj); return obj; '''), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -403,14 +403,14 @@ void *array_data[] = {NULL, NULL}; return PyUFunc_FromFuncAndDataAndSignature(funcs, array_data, types, 1, 1, 1, PyUFunc_None, - "float_3x3", - "a ufunc that tests a more complicated signature", + "float_3x3", + "a ufunc that tests a more complicated signature", 0, "(m,m)->(m,m)"); """), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -480,7 +480,7 @@ res += +10; *((float *)args[1]) = res; }; - + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -127,12 +127,12 @@ test_compare(1, 2) test_compare(2, 2) test_compare('2', '1') - + w_i = space.wrap(1) assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() - + def test_IsInstance(self, space, api): assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 @@ -165,7 +165,7 @@ return File""") w_f = space.call_function(w_File) assert api.PyObject_AsFileDescriptor(w_f) == 42 - + def test_hash(self, space, api): assert api.PyObject_Hash(space.wrap(72)) == 72 assert api.PyObject_Hash(space.wrap(-1)) == -1 @@ -250,7 +250,7 @@ if (copy != orig) PyObject_Free(copy); PyObject_Free(orig); - return ret; + return ret; """)]) x = module.realloctest() assert x == 'hello world\x00' @@ -425,7 +425,6 @@ """ Py_buffer buf; PyObject *str = PyString_FromString("hello, world."); - PyObject *result; if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13, 1, PyBUF_WRITABLE)) { diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -1,5 +1,7 @@ +from pypy.conftest import option from pypy.module.cpyext.api import fopen, fclose, fwrite from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.object import Py_PRINT_RAW from rpython.rtyper.lltypesystem import rffi, lltype from rpython.tool.udir import udir @@ -111,3 +113,34 @@ out, err = capfd.readouterr() out = out.replace('\r\n', '\n') assert out == " 1 23\n" + + +class AppTestPyFile(AppTestCpythonExtensionBase): + + def setup_class(cls): + from rpython.tool.udir import udir + if option.runappdirect: + cls.w_udir = str(udir) + else: + cls.w_udir = cls.space.wrap(str(udir)) + + def test_file_tell(self): + module = self.import_extension('foo', [ + ("get_c_tell", "METH_O", + """ + FILE * fp = PyFile_AsFile(args); + if (fp == NULL) + return PyLong_FromLong(0); + return PyLong_FromLong(ftell(fp)); + """), + ]) + filename = self.udir + "/_test_file" + with open(filename, 'w') as fid: + fid.write('3' * 122) + with open(filename, 'r') as fid: + s = fid.read(80) + t_py = fid.tell() + assert t_py == 80 + t_c = module.get_c_tell(fid) + assert t_c == t_py + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -130,7 +130,7 @@ module = self.import_extension('foo', [ ("run", "METH_NOARGS", """ - long prev, next; + long prev; PyObject *t = PyTuple_New(1); prev = Py_True->ob_refcnt; Py_INCREF(Py_True); diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,8 +5,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr -import sys - class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): import sys @@ -737,7 +735,6 @@ """ IntLikeObject *intObj; int intval; - PyObject *name; if (!PyArg_ParseTuple(args, "i", &intval)) return NULL; @@ -897,7 +894,7 @@ module.footype("X", (object,), {}) def test_app_subclass_of_c_type(self): - # on cpython, the size changes (6 bytes added) + import sys module = self.import_module(name='foo') size = module.size_of_instances(module.fooType) class f1(object): @@ -907,7 +904,11 @@ class bar(f1, f2): pass assert bar.__base__ is f2 - assert module.size_of_instances(bar) == size + # On cpython, the size changes. + if '__pypy__' in sys.builtin_module_names: + assert module.size_of_instances(bar) == size + else: + assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): module = self.import_module(name='foo') @@ -1058,7 +1059,6 @@ module = self.import_extension('foo', [ ("getMetaClass", "METH_NOARGS", ''' - PyObject *obj; FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT; FooType_Type.tp_base = &PyType_Type; if (PyType_Ready(&FooType_Type) < 0) return NULL; diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -35,7 +35,7 @@ ("test_GetSize_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyUnicode_GetSize(f); + PyUnicode_GetSize(f); Py_DECREF(f); return NULL; @@ -57,7 +57,6 @@ """ PyObject *s, *t; Py_UNICODE* c; - Py_ssize_t len; s = PyUnicode_FromUnicode(NULL, 4); if (s == NULL) @@ -84,7 +83,7 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyUnicodeObject*)obj)->hash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ]) @@ -234,13 +233,13 @@ w_res = api.PyUnicode_AsUTF8String(w_u) assert space.type(w_res) is space.w_str assert space.unwrap(w_res) == 'sp\tm' - + def test_decode_utf8(self, space, api): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) w_u = api.PyUnicode_DecodeUTF8(u, 5, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == u'sp\x134m' - + w_u = api.PyUnicode_DecodeUTF8(u, 2, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == 'sp' @@ -405,7 +404,7 @@ ustr = "abcdef" w_ustr = space.wrap(ustr.decode("ascii")) result = api.PyUnicode_AsASCIIString(w_ustr) - + assert space.eq_w(space.wrap(ustr), result) w_ustr = space.wrap(u"abcd\xe9f") diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -142,6 +142,7 @@ ref = rffi.cast(PyTupleObject, ref) size = ref.c_ob_size if index < 0 or index >= size: + decref(space, py_obj) raise oefmt(space.w_IndexError, "tuple assignment index out of range") old_ref = ref.c_ob_item[index] ref.c_ob_item[index] = py_obj # consumes a reference diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -657,6 +657,8 @@ pto.c_tp_dealloc = llhelper( subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) + if space.is_w(w_type, space.w_str): + pto.c_tp_itemsize = 1 # buffer protocol setup_buffer_procs(space, w_type, pto) @@ -695,6 +697,8 @@ if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize + if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize: + pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize # will be filled later on with the correct value # may not be 0 diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -77,7 +77,9 @@ """ py_uni = rffi.cast(PyUnicodeObject, py_obj) s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length) - w_obj = space.wrap(s) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type) + w_obj.__init__(s) py_uni.c_hash = space.hash_w(w_obj) track_reference(space, py_obj, w_obj) return w_obj diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py --- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py +++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py @@ -163,14 +163,10 @@ guard_not_invalidated(descr=...) i32 = float_ne(f31, 0.000000) guard_true(i32, descr=...) - i34 = getarrayitem_raw_i(#, #, descr=) # XXX what are these? - guard_value(i34, #, descr=...) # XXX don't appear in - i35 = getarrayitem_raw_i(#, #, descr=) # XXX equiv test_zjit i36 = int_add(i24, 1) i37 = int_add(i29, 8) i38 = int_ge(i36, i30) guard_false(i38, descr=...) - guard_value(i35, #, descr=...) # XXX jump(..., descr=...) """) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 3, 1, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 3, 2, "alpha", 0) #XXX # sync patchlevel.h import pypy diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -18,6 +18,7 @@ from pypy.objspace.std.unicodeobject import ( decode_object, unicode_from_encoded_object, unicode_from_string, getdefaultencoding) +from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT class W_AbstractBytesObject(W_Root): @@ -30,12 +31,26 @@ return True if self.user_overridden_class or w_other.user_overridden_class: return False - return space.str_w(self) is space.str_w(w_other) + s1 = space.str_w(self) + s2 = space.str_w(w_other) + if len(s2) > 1: + return s1 is s2 + else: # strings of len <= 1 are unique-ified + return s1 == s2 def immutable_unique_id(self, space): if self.user_overridden_class: return None - return space.wrap(compute_unique_id(space.str_w(self))) + s = space.str_w(self) + if len(s) > 1: + uid = compute_unique_id(s) + else: # strings of len <= 1 are unique-ified + if len(s) == 1: + base = ord(s[0]) # base values 0-255 + else: + base = 256 # empty string: base value 256 + uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) def unicode_w(self, space): # Use the default encoding. diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -6,6 +6,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.unicodeobject import W_UnicodeObject +from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT from rpython.rlib.objectmodel import r_dict from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash @@ -575,6 +576,23 @@ class W_FrozensetObject(W_BaseSetObject): hash = 0 + def is_w(self, space, w_other): + if not isinstance(w_other, W_FrozensetObject): + return False + if self is w_other: + return True + if self.user_overridden_class or w_other.user_overridden_class: + return False + # empty frozensets are unique-ified + return 0 == w_other.length() == self.length() + + def immutable_unique_id(self, space): + if self.user_overridden_class or self.length() > 0: + return None + # empty frozenset: base value 259 + uid = (259 << IDTAG_SHIFT) | IDTAG_SPECIAL + return space.wrap(uid) + def _newobj(self, space, w_iterable): """Make a new frozenset by taking ownership of 'w_iterable'.""" if type(self) is W_FrozensetObject: diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -162,7 +162,8 @@ buffer = _get_buffer(space, w_sub) res = count(value, buffer, start, end) - return space.wrap(max(res, 0)) + assert res >= 0 + return space.wrap(res) def descr_decode(self, space, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodeobject import ( diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -211,6 +211,7 @@ check(bytearray('abc').replace('b', bytearray('d')), 'adc') check(bytearray('abc').replace('b', 'd'), 'adc') + check(bytearray('').replace('a', 'ab'), '') check(bytearray('abc').upper(), 'ABC') check(bytearray('ABC').lower(), 'abc') diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -186,17 +186,36 @@ def test_id_on_strs(self): if self.appdirect: skip("cannot run this test as apptest") - u = u"a" - assert id(self.unwrap_wrap_unicode(u)) == id(u) - s = "a" - assert id(self.unwrap_wrap_str(s)) == id(s) + for u in [u"", u"a", u"aa"]: + assert id(self.unwrap_wrap_unicode(u)) == id(u) + s = str(u) + assert id(self.unwrap_wrap_str(s)) == id(s) + # + assert id('') == (256 << 4) | 11 # always + assert id(u'') == (257 << 4) | 11 + assert id('a') == (ord('a') << 4) | 11 + assert id(u'\u1234') == ((~0x1234) << 4) | 11 + + def test_id_of_tuples(self): + l = [] + x = (l,) + assert id(x) != id((l,)) # no caching at all + if self.appdirect: + skip("cannot run this test as apptest") + assert id(()) == (258 << 4) | 11 # always + + def test_id_of_frozensets(self): + x = frozenset([4]) + assert id(x) != id(frozenset([4])) # no caching at all + if self.appdirect: + skip("cannot run this test as apptest") + assert id(frozenset()) == (259 << 4) | 11 # always + assert id(frozenset([])) == (259 << 4) | 11 # always def test_identity_vs_id_primitives(self): - if self.cpython_apptest: - skip("cpython behaves differently") import sys - l = range(-10, 10) - for i in range(10): + l = range(-10, 10, 2) + for i in [0, 1, 3]: l.append(float(i)) l.append(i + 0.1) l.append(long(i)) @@ -206,18 +225,15 @@ l.append(i - 1j) l.append(1 + i * 1j) l.append(1 - i * 1j) - s = str(i) - l.append(s) - u = unicode(s) - l.append(u) + l.append((i,)) + l.append(frozenset([i])) l.append(-0.0) l.append(None) l.append(True) l.append(False) - s = "s" - l.append(s) - s = u"s" - l.append(s) + l.append(()) + l.append(tuple([])) + l.append(frozenset()) for i, a in enumerate(l): for b in l[i:]: @@ -228,21 +244,18 @@ def test_identity_vs_id_str(self): if self.appdirect: skip("cannot run this test as apptest") - import sys - l = range(-10, 10) - for i in range(10): - s = str(i) + l = [] + def add(s, u): l.append(s) l.append(self.unwrap_wrap_str(s)) - u = unicode(s) + l.append(s[:1] + s[1:]) l.append(u) l.append(self.unwrap_wrap_unicode(u)) From pypy.commits at gmail.com Mon Jun 27 05:14:09 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 02:14:09 -0700 (PDT) Subject: [pypy-commit] pypy default: Silence the warning: old-style non-light finalizers are still ok Message-ID: <5770ee61.cbc71c0a.84480.6c09@mx.google.com> Author: Armin Rigo Branch: Changeset: r85395:3d35185f9c4e Date: 2016-06-27 11:15 +0200 http://bitbucket.org/pypy/pypy/changeset/3d35185f9c4e/ Log: Silence the warning: old-style non-light finalizers are still ok diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py --- a/rpython/translator/backendopt/finalizer.py +++ b/rpython/translator/backendopt/finalizer.py @@ -36,8 +36,8 @@ raise FinalizerError(msg) else: result = self.analyze_direct_call(graph) - if result is self.top_result(): - log.red('old-style non-light finalizer: %r' % (graph,)) + #if result is self.top_result(): + # log.red('old-style non-light finalizer: %r' % (graph,)) return result def analyze_simple_operation(self, op, graphinfo): From pypy.commits at gmail.com Mon Jun 27 05:20:39 2016 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 27 Jun 2016 02:20:39 -0700 (PDT) Subject: [pypy-commit] pypy default: remove mentions of translate.py from the docs Message-ID: <5770efe7.10371c0a.f40a7.3fdf@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r85396:bfe0ecebb5c0 Date: 2016-06-27 11:17 +0200 http://bitbucket.org/pypy/pypy/changeset/bfe0ecebb5c0/ Log: remove mentions of translate.py from the docs diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/rpython/doc/getting-started.rst b/rpython/doc/getting-started.rst --- a/rpython/doc/getting-started.rst +++ b/rpython/doc/getting-started.rst @@ -142,8 +142,8 @@ Translating Full Programs ~~~~~~~~~~~~~~~~~~~~~~~~~ -To translate full RPython programs, there is the script ``translate.py`` in -:source:`rpython/translator/goal`. Examples for this are a slightly changed version of +To translate full RPython programs, there is the script ``bin/rpython`` in +:source:`rpython/bin/`. Examples for this are a slightly changed version of Pystone:: python bin/rpython translator/goal/targetrpystonedalone From pypy.commits at gmail.com Mon Jun 27 08:59:02 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 05:59:02 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Finalizers: recording Message-ID: <57712316.09f6c20a.f91f0.3ec2@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85397:0477d9c6b062 Date: 2016-06-27 15:00 +0200 http://bitbucket.org/pypy/pypy/changeset/0477d9c6b062/ Log: Finalizers: recording diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -80,6 +80,9 @@ RPY_EXTERN void (*boehm_fq_trigger[])(void); int boehm_gc_finalizer_lock = 0; +void boehm_gc_finalizer_notifier(void); + +#ifndef RPY_REVERSE_DEBUGGER void boehm_gc_finalizer_notifier(void) { int i; @@ -101,6 +104,10 @@ boehm_gc_finalizer_lock--; } +#else +/* see revdb.c */ +RPY_EXTERN void rpy_reverse_db_next_dead(void *); +#endif static void mem_boehm_ignore(char *msg, GC_word arg) { @@ -128,12 +135,17 @@ void *boehm_fq_next_dead(struct boehm_fq_s **fqueue) { struct boehm_fq_s *node = *fqueue; + void *result; if (node != NULL) { *fqueue = node->next; - return node->obj; + result = node->obj; } else - return NULL; + result = NULL; +#ifdef RPY_REVERSE_DEBUGGER + rpy_reverse_db_next_dead(result); +#endif + return result; } #endif /* BOEHM GC */ diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -25,7 +25,7 @@ #define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) #define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) -//#define ASYNC_FINALIZER ((int16_t)0xff46) +#define ASYNC_FINALIZER_TRIGGER ((int16_t)0xff46) typedef struct { @@ -82,6 +82,18 @@ check_at_end(stop_points); } +static void record_stop_point(void); +static void replay_stop_point(void); + +RPY_EXTERN +void rpy_reverse_db_stop_point(void) +{ + if (!RPY_RDB_REPLAY) + record_stop_point(); + else + replay_stop_point(); +} + /* ------------------------------------------------------------ */ /* Recording mode */ @@ -145,23 +157,81 @@ static void flush_buffer(void) { /* write the current buffer content to the OS */ - ssize_t size = rpy_revdb.buf_p - rpy_rev_buffer; + ssize_t full_size = rpy_revdb.buf_p - rpy_rev_buffer; rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t); if (rpy_rev_fileno >= 0) - write_all(rpy_rev_buffer, size); + write_all(rpy_rev_buffer, full_size); +} + +static ssize_t current_packet_size(void) +{ + return rpy_revdb.buf_p - (rpy_rev_buffer + sizeof(int16_t)); } RPY_EXTERN void rpy_reverse_db_flush(void) { - ssize_t packet_size = rpy_revdb.buf_p - (rpy_rev_buffer + sizeof(int16_t)); - if (packet_size != 0) { - assert(0 < packet_size && packet_size <= 32767); - *(int16_t *)rpy_rev_buffer = packet_size; + ssize_t content_size = current_packet_size(); + if (content_size != 0) { + assert(0 < content_size && content_size <= 32767); + *(int16_t *)rpy_rev_buffer = content_size; flush_buffer(); } } +/* +RPY_EXTERN +void rpy_reverse_db_register_finalizer(void *obj, void (*finalizer)(void *)) +{ + ...; +} +*/ + +void boehm_gc_finalizer_notifier(void) +{ + /* This is called by Boehm when there are pending finalizers. + They are only invoked when we call GC_invoke_finalizers(), + which we only do at stop points in the case of revdb. + */ + assert(rpy_revdb.stop_point_break <= rpy_revdb.stop_point_seen + 1); + rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1; +} + +static void record_stop_point(void) +{ + /* Invoke the finalizers now. This will call boehm_fq_callback(), + which will enqueue the objects in the correct FinalizerQueue. + Then, call boehm_fq_trigger(), which calls finalizer_trigger(). + */ + int i; + rpy_reverse_db_flush(); + + GC_invoke_finalizers(); + i = 0; + while (boehm_fq_trigger[i]) + boehm_fq_trigger[i++](); + + /* This should all be done without emitting anything to the rdb + log. We check that, and emit just a ASYNC_FINALIZER_TRIGGER. + */ + if (current_packet_size() != 0) { + fprintf(stderr, + "record_stop_point emitted unexpectedly to the rdb log\n"); + exit(1); + } + *(int16_t *)rpy_rev_buffer = ASYNC_FINALIZER_TRIGGER; + memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t)); + rpy_revdb.buf_p += sizeof(uint64_t); + flush_buffer(); +} + +RPY_EXTERN +void rpy_reverse_db_next_dead(void *result) +{ + int64_t uid = result ? ((struct pypy_header0 *)result)->h_uid : -1; + RPY_REVDB_EMIT(/* nothing */, int64_t _e, uid); +} + RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj) { @@ -828,8 +898,7 @@ rpy_revdb.watch_enabled = any_watch_point; } -RPY_EXTERN -void rpy_reverse_db_stop_point(void) +static void replay_stop_point(void) { while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { save_state(); diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -112,6 +112,9 @@ #define OP_REVDB_WEAKREF_DEREF(weakref, r) \ r = rpy_reverse_db_weakref_deref(weakref) +#define OP_REVDB_CALL_DESTRUCTOR(obj, r) \ + rpy_reverse_db_call_destructor(obj) + RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN char *rpy_reverse_db_fetch(const char *file, int line); RPY_EXTERN void rpy_reverse_db_stop_point(void); @@ -125,5 +128,7 @@ RPY_EXTERN void rpy_reverse_db_watch_restore_state(bool_t any_watch_point); RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target); RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref); +//RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); +RPY_EXTERN void rpy_reverse_db_next_dead(void *result); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -44,6 +44,21 @@ assert self.cur <= self.current_packet_end return result + def is_special_packet(self): + if self.current_packet_end != self.cur: + assert self.current_packet_end > self.cur + return False + next_header = struct.unpack_from('h', self.buffer, self.cur)[0] + return (next_header & 0xFF00) == 0xFF00 + + def special_packet(self, expected, fmt): + assert self.current_packet_end == self.cur + next_id = self.read1('h') + assert next_id == expected + p = self.cur + self.cur = self.current_packet_end = p + struct.calcsize(fmt) + return struct.unpack_from(fmt, self.buffer, p) + def read_check_argv(self, expected): assert self.argc == len(expected) for i in range(self.argc): diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -2,6 +2,7 @@ from rpython.rlib import revdb, rgc from rpython.rlib.debug import debug_print from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rarithmetic import intmask from rpython.translator.revdb.message import * from rpython.translator.revdb.test.test_basic import BaseRecordingTests from rpython.translator.revdb.test.test_basic import InteractiveTests @@ -22,6 +23,8 @@ WEAKREF_AFTERWARDS_DEAD = chr(0xf2) WEAKREF_AFTERWARDS_ALIVE = chr(0xeb) +ASYNC_FINALIZER_TRIGGER = 0xff46 - 2**16 + class TestRecording(BaseRecordingTests): @@ -80,6 +83,8 @@ lst = [X() for i in range(3000)] for i in range(3000): lst[i] = None + if i % 300 == 150: + rgc.collect() revdb.stop_point() return 9 self.compile(main, [], backendopt=False) @@ -88,6 +93,124 @@ x = rdb.next('q'); assert x == 3000 # number of stop points assert rdb.done() + def test_finalizer_queue(self): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.translator.tool.cbuild import ExternalCompilationInfo + eci = ExternalCompilationInfo( + pre_include_bits=["#define foobar(x) x\n"]) + foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, + compilation_info=eci) + class Glob: + pass + glob = Glob() + class X: + pass + class MyFinalizerQueue(rgc.FinalizerQueue): + Class = X + def finalizer_trigger(self): + glob.ping = True + fq = MyFinalizerQueue() + # + def main(argv): + glob.ping = False + lst1 = [X() for i in range(256)] + lst = [X() for i in range(3000)] + for i, x in enumerate(lst): + x.baz = i + fq.register_finalizer(x) + for i in range(3000): + lst[i] = None + if i % 300 == 150: + rgc.collect() + revdb.stop_point() + j = i + glob.ping * 1000000 + assert foobar(j) == j + if glob.ping: + glob.ping = False + total = 0 + while True: + x = fq.next_dead() + if x is None: + break + total = intmask(total * 3 + x.baz) + assert foobar(total) == total + keepalive_until_here(lst1) + return 9 + self.compile(main, [], backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + uid_seen = set() + totals = [] + for i in range(3000): + triggered = False + if rdb.is_special_packet(): + time, = rdb.special_packet(ASYNC_FINALIZER_TRIGGER, 'q') + assert time == i + 1 + triggered = True + j = rdb.next() + assert j == i + 1000000 * triggered + if triggered: + lst = [] + while True: + uid = intmask(rdb.next()) + if uid == -1: + break + assert uid > 0 and uid not in uid_seen + uid_seen.add(uid) + lst.append(uid) + totals.append((lst, intmask(rdb.next()))) + x = rdb.next('q'); assert x == 3000 # number of stop points + # + assert 1500 <= len(uid_seen) <= 3000 + d = dict(zip(sorted(uid_seen), range(len(uid_seen)))) + for lst, expected in totals: + total = 0 + for uid in lst: + total = intmask(total * 3 + d[uid]) + assert total == expected + + def test_finalizer_recorded(self): + py.test.skip("in-progress") + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.translator.tool.cbuild import ExternalCompilationInfo + eci = ExternalCompilationInfo( + pre_include_bits=["#define foobar(x) x\n"]) + foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, + compilation_info=eci) + class Glob: + pass + glob = Glob() + class X: + def __del__(self): + glob.count += 1 + def main(argv): + glob.count = 0 + lst = [X() for i in range(3000)] + x = -1 + for i in range(3000): + lst[i] = None + if i % 300 == 150: + rgc.collect() + revdb.stop_point() + x = glob.count + assert foobar(x) == x + print x + return 9 + self.compile(main, [], backendopt=False) + out = self.run('Xx') + assert 1500 < int(out) <= 3000 + rdb = self.fetch_rdb([self.exename, 'Xx']) + counts = [rdb.next() for i in range(3000)] + assert counts[0] >= 0 + for i in range(len(counts)-1): + assert counts[i] <= counts[i + 1] + assert counts[-1] == int(out) + # write() call + x = rdb.next(); assert x == len(out) + x = rdb.next('i'); assert x == 0 # errno + x = rdb.next('q'); assert x == 3000 # number of stop points + assert rdb.done() + class TestReplaying(InteractiveTests): expected_stop_points = 1 From pypy.commits at gmail.com Mon Jun 27 10:00:01 2016 From: pypy.commits at gmail.com (plan_rich) Date: Mon, 27 Jun 2016 07:00:01 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: refactoring of vec_load/vec_store, now the same simplification (as e.g. raw_load -> gc_load) is done Message-ID: <57713161.d11b1c0a.15db7.0482@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85398:3a5a728d551e Date: 2016-06-27 15:59 +0200 http://bitbucket.org/pypy/pypy/changeset/3a5a728d551e/ Log: refactoring of vec_load/vec_store, now the same simplification (as e.g. raw_load -> gc_load) is done arch dependant code now moved the backend diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -35,6 +35,7 @@ # can an ISA instruction handle a factor to the offset? load_supported_factors = (1,) + vector_ext = None vector_extension = False vector_register_size = 0 # in bytes vector_horizontal_operations = False diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -156,32 +156,12 @@ index_box = op.getarg(1) self.emit_gc_load_or_indexed(op, ptr_box, index_box, itemsize, itemsize, ofs, sign) - def handle_rawload(self, op): - itemsize, ofs, sign = unpack_arraydescr(op.getdescr()) - ptr_box = op.getarg(0) - index_box = op.getarg(1) - self.emit_gc_load_or_indexed(op, ptr_box, index_box, itemsize, 1, ofs, sign) - def _emit_mul_if_factor_offset_not_supported(self, index_box, factor, offset): - # Returns (factor, offset, index_box) where index_box is either - # a non-constant BoxInt or None. - if isinstance(index_box, ConstInt): - return 1, index_box.value * factor + offset, None - else: - if factor != 1 and factor not in self.cpu.load_supported_factors: - # the factor is supported by the cpu - # x & (x - 1) == 0 is a quick test for power of 2 - assert factor > 0 - if (factor & (factor - 1)) == 0: - index_box = ResOperation(rop.INT_LSHIFT, - [index_box, ConstInt(highest_bit(factor))]) - else: - index_box = ResOperation(rop.INT_MUL, - [index_box, ConstInt(factor)]) - self.emit_op(index_box) - factor = 1 - return factor, offset, index_box + factor, offset, index_box = cpu_simplify_scale(self.cpu, indexbox, factor, offset) + if index_box: + self.emit_op(index_box) + return factor, offset, index_box def emit_gc_load_or_indexed(self, op, ptr_box, index_box, itemsize, factor, offset, sign, type='i'): @@ -985,3 +965,23 @@ self._newops.append(load_op) self.gcrefs_recently_loaded[index] = load_op return load_op + +def cpu_simplify_scale(cpu, index_box, factor, offset): + # Returns (factor, offset, index_box, [ops]) where index_box is either + # a non-constant BoxInt or None. + if isinstance(index_box, ConstInt): + return 1, index_box.value * factor + offset, None + else: + if factor != 1 and factor not in cpu.load_supported_factors: + # the factor is supported by the cpu + # x & (x - 1) == 0 is a quick test for power of 2 + assert factor > 0 + if (factor & (factor - 1)) == 0: + index_box = ResOperation(rop.INT_LSHIFT, + [index_box, ConstInt(highest_bit(factor))]) + else: + index_box = ResOperation(rop.INT_MUL, + [index_box, ConstInt(factor)]) + factor = 1 + return factor, offset, index_box + diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/vector_ext.py @@ -0,0 +1,235 @@ +from rpython.jit.backend.llsupport.descr import (unpack_arraydescr, + unpack_fielddescr, unpack_interiorfielddescr) +from rpython.rlib.objectmodel import specialize, always_inline +from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT) +from rpython.jit.metainterp.resoperation import rop + +class TypeRestrict(object): + ANY_TYPE = '\x00' + ANY_SIZE = -1 + ANY_SIGN = -1 + ANY_COUNT = -1 + SIGNED = 1 + UNSIGNED = 0 + + def __init__(self, + type=ANY_TYPE, + bytesize=ANY_SIZE, + count=ANY_SIGN, + sign=ANY_COUNT): + self.type = type + self.bytesize = bytesize + self.sign = sign + self.count = count + + @always_inline + def any_size(self): + return self.bytesize == TypeRestrict.ANY_SIZE + + @always_inline + def any_count(self): + return self.count == TypeRestrict.ANY_COUNT + + def check(self, value): + vecinfo = forwarded_vecinfo(value) + assert vecinfo.datatype != '\x00' + if self.type != TypeRestrict.ANY_TYPE: + if self.type != vecinfo.datatype: + msg = "type mismatch %s != %s" % \ + (self.type, vecinfo.datatype) + failnbail_transformation(msg) + assert vecinfo.bytesize > 0 + if not self.any_size(): + if self.bytesize != vecinfo.bytesize: + msg = "bytesize mismatch %s != %s" % \ + (self.bytesize, vecinfo.bytesize) + failnbail_transformation(msg) + assert vecinfo.count > 0 + if self.count != TypeRestrict.ANY_COUNT: + if vecinfo.count < self.count: + msg = "count mismatch %s < %s" % \ + (self.count, vecinfo.count) + failnbail_transformation(msg) + if self.sign != TypeRestrict.ANY_SIGN: + if bool(self.sign) == vecinfo.sign: + msg = "sign mismatch %s < %s" % \ + (self.sign, vecinfo.sign) + failnbail_transformation(msg) + + def max_input_count(self, count): + """ How many """ + if self.count != TypeRestrict.ANY_COUNT: + return self.count + return count + +class OpRestrict(object): + def __init__(self, argument_restris): + self.argument_restrictions = argument_restris + + def check_operation(self, state, pack, op): + pass + + def crop_vector(self, op, newsize, size): + return newsize, size + + def must_crop_vector(self, op, index): + restrict = self.argument_restrictions[index] + vecinfo = forwarded_vecinfo(op.getarg(index)) + size = vecinfo.bytesize + newsize = self.crop_to_size(op, index) + return not restrict.any_size() and newsize != size + + @always_inline + def crop_to_size(self, op, index): + restrict = self.argument_restrictions[index] + return restrict.bytesize + + def opcount_filling_vector_register(self, op, vec_reg_size): + """ How many operations of that kind can one execute + with a machine instruction of register size X? + """ + if op.is_typecast(): + if op.casts_down(): + size = op.cast_input_bytesize(vec_reg_size) + return size // op.cast_from_bytesize() + else: + return vec_reg_size // op.cast_to_bytesize() + vecinfo = forwarded_vecinfo(op) + return vec_reg_size // vecinfo.bytesize + +class GuardRestrict(OpRestrict): + def opcount_filling_vector_register(self, op, vec_reg_size): + arg = op.getarg(0) + vecinfo = forwarded_vecinfo(arg) + return vec_reg_size // vecinfo.bytesize + +class LoadRestrict(OpRestrict): + def check_operation(self, state, pack, op): + opnum = op.getopnum() + if rop.is_getarrayitem(opnum) or \ + opnum in (rop.GETARRAYITEM_RAW_I, rop.GETARRAYITEM_RAW_F): + itemsize, ofs, sign = unpack_arraydescr(op.getdescr()) + index_box = op.getarg(1) + _, _, changed = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs) + if changed is not index_box: + state.oplist.append(changed) + op.setarg(1, changed) + + def opcount_filling_vector_register(self, op, vec_reg_size): + assert rop.is_primitive_load(op.opnum) + descr = op.getdescr() + return vec_reg_size // descr.get_item_size_in_bytes() + +class StoreRestrict(OpRestrict): + def __init__(self, argument_restris): + self.argument_restrictions = argument_restris + + def check_operation(self, state, pack, op): + opnum = op.getopnum() + if opnum in (rop.SETARRAYITEM_GC, rop.SETARRAYITEM_RAW): + itemsize, basesize, _ = unpack_arraydescr(op.getdescr()) + index_box = op.getarg(1) + _, _, changed = cpu_simplify_scale(index_box, itemsize, basesize) + if changed is not index_box: + state.oplist.append(changed) + op.setarg(1, changed) + + def must_crop_vector(self, op, index): + vecinfo = forwarded_vecinfo(op.getarg(index)) + bytesize = vecinfo.bytesize + return self.crop_to_size(op, index) != bytesize + + @always_inline + def crop_to_size(self, op, index): + # there is only one parameter that needs to be transformed! + descr = op.getdescr() + return descr.get_item_size_in_bytes() + + def opcount_filling_vector_register(self, op, vec_reg_size): + assert rop.is_primitive_store(op.opnum) + descr = op.getdescr() + return vec_reg_size // descr.get_item_size_in_bytes() + +class OpMatchSizeTypeFirst(OpRestrict): + def check_operation(self, state, pack, op): + i = 0 + infos = [forwarded_vecinfo(o) for o in op.getarglist()] + arg0 = op.getarg(i) + while arg0.is_constant() and i < op.numargs(): + i += 1 + arg0 = op.getarg(i) + vecinfo = forwarded_vecinfo(arg0) + bytesize = vecinfo.bytesize + datatype = vecinfo.datatype + + for arg in op.getarglist(): + if arg.is_constant(): + continue + curvecinfo = forwarded_vecinfo(arg) + if curvecinfo.bytesize != bytesize: + raise NotAVectorizeableLoop() + if curvecinfo.datatype != datatype: + raise NotAVectorizeableLoop() + +TR_ANY = TypeRestrict() +TR_ANY_FLOAT = TypeRestrict(FLOAT) +TR_ANY_INTEGER = TypeRestrict(INT) +TR_FLOAT_2 = TypeRestrict(FLOAT, 4, 2) +TR_DOUBLE_2 = TypeRestrict(FLOAT, 8, 2) +TR_INT32_2 = TypeRestrict(INT, 4, 2) + +OR_MSTF_I = OpMatchSizeTypeFirst([TR_ANY_INTEGER, TR_ANY_INTEGER]) +OR_MSTF_F = OpMatchSizeTypeFirst([TR_ANY_FLOAT, TR_ANY_FLOAT]) +STORE_RESTRICT = StoreRestrict([None, None, TR_ANY]) +LOAD_RESTRICT = LoadRestrict([]) +GUARD_RESTRICT = GuardRestrict([TR_ANY_INTEGER]) + + +class VectorExt(object): + + # note that the following definition is x86 arch specific + TR_MAPPING = { + rop.VEC_INT_ADD: OR_MSTF_I, + rop.VEC_INT_SUB: OR_MSTF_I, + rop.VEC_INT_MUL: OR_MSTF_I, + rop.VEC_INT_AND: OR_MSTF_I, + rop.VEC_INT_OR: OR_MSTF_I, + rop.VEC_INT_XOR: OR_MSTF_I, + rop.VEC_INT_EQ: OR_MSTF_I, + rop.VEC_INT_NE: OR_MSTF_I, + + rop.VEC_FLOAT_ADD: OR_MSTF_F, + rop.VEC_FLOAT_SUB: OR_MSTF_F, + rop.VEC_FLOAT_MUL: OR_MSTF_F, + rop.VEC_FLOAT_TRUEDIV: OR_MSTF_F, + rop.VEC_FLOAT_ABS: OpRestrict([TR_ANY_FLOAT]), + rop.VEC_FLOAT_NEG: OpRestrict([TR_ANY_FLOAT]), + + rop.VEC_STORE: STORE_RESTRICT, + + rop.VEC_LOAD_I: LOAD_RESTRICT, + rop.VEC_LOAD_F: LOAD_RESTRICT, + + rop.VEC_GUARD_TRUE: GUARD_RESTRICT, + rop.VEC_GUARD_FALSE: GUARD_RESTRICT, + + ## irregular + rop.VEC_INT_SIGNEXT: OpRestrict([TR_ANY_INTEGER]), + + rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpRestrict([TR_DOUBLE_2]), + # weird but the trace will store single floats in int boxes + rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpRestrict([TR_INT32_2]), + rop.VEC_CAST_FLOAT_TO_INT: OpRestrict([TR_DOUBLE_2]), + rop.VEC_CAST_INT_TO_FLOAT: OpRestrict([TR_INT32_2]), + + rop.VEC_FLOAT_EQ: OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]), + rop.VEC_FLOAT_NE: OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]), + rop.VEC_INT_IS_TRUE: OpRestrict([TR_ANY_INTEGER,TR_ANY_INTEGER]), + } + + def get_operation_restriction(self, op): + res = self.TR_MAPPING.get(op.vector, None) + if not res: + failnbail_transformation("could not get OpRestrict for " + str(op)) + return res + diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -62,8 +62,9 @@ XFX = Form("CRM", "rS", "XO1") XLL = Form("LL", "XO1") XX1 = Form("fvrT", "rA", "rB", "XO1") -XX2 = Form("fvrT", "fvrB", "XO5") +XX2 = Form("fvrT", "fvrB", "XO6") XX3 = Form("fvrT", "fvrA", "fvrB", "XO9") +XX3_2 = Form("fvrT", "fvrA", "fvrB", "OE", "XO11") XV = Form("ivrT", "rA", "rB", "XO1") VX = Form("ivrT", "ivrA", "ivrB", "XO8") VXI = Form("ivrT", "SIM", "XO8") @@ -611,14 +612,22 @@ # div xvdivdp = XX3(60, XO9=102) xvdivsp = XX3(60, XO9=88) + # cmp + xvcmpeqdp = XX3_2(60, XO11=99, OE=0) + xvcmpeqdpx = XX3_2(60, XO11=99, OE=1) + xvcmpeqsp = XX3_2(60, XO11=67, OE=0) + xvcmpeqspx = XX3_2(60, XO11=67, OE=1) + + # logical and and complement + xxlandc = XX3(60, XO9=138) # neg - xvnegdp = XX2(60, XO5=505) - xvabssp = XX2(60, XO5=441) + xvnegdp = XX2(60, XO6=505) + xvnegsp = XX2(60, XO6=441) # abs - xvabsdp = XX2(60, XO5=473) - xvabssp = XX2(60, XO5=409) + xvabsdp = XX2(60, XO6=473) + xvabssp = XX2(60, XO6=409) # INTEGER # ------- diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py --- a/rpython/jit/backend/ppc/ppc_field.py +++ b/rpython/jit/backend/ppc/ppc_field.py @@ -64,6 +64,7 @@ "XO8": (21, 31), "XO9": (21, 28), "XO10": (26, 31), + "XO11": (22, 28), "LL": ( 9, 10), "SIM": (11, 15), } diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py --- a/rpython/jit/backend/ppc/runner.py +++ b/rpython/jit/backend/ppc/runner.py @@ -3,6 +3,7 @@ from rpython.rlib import rgc from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU +from rpython.jit.backend.ppc.vector_ext import AltiVectorExt from rpython.jit.backend.ppc.ppc_assembler import AssemblerPPC from rpython.jit.backend.ppc.arch import WORD from rpython.jit.backend.ppc.codebuilder import PPCBuilder @@ -11,6 +12,7 @@ class PPC_CPU(AbstractLLCPU): + vector_ext = None vector_extension = False # may be set to true in setup vector_register_size = 16 vector_horizontal_operations = False @@ -47,6 +49,7 @@ def setup_once(self): self.assembler.setup_once() if detect_vsx(): + self.vector_ext = AltiVectorExt() self.vector_extension = True # ??? self.vector_horizontal_operations = True diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -12,6 +12,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.jit.backend.ppc.locations import imm from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN +from rpython.jit.backend.llsupport.vector_ext import VectorExt def not_implemented(msg): msg = '[ppc/vector_ext] %s\n' % msg @@ -19,6 +20,9 @@ llop.debug_print(lltype.Void, msg) raise NotImplementedError(msg) +class AltiVectorExt(VectorExt): + pass + class VectorAssembler(object): _mixin_ = True @@ -348,22 +352,20 @@ # # entries before) become ones # self.mc.PCMPEQ(loc, temp, sizeloc.value) - #def genop_vec_float_eq(self, op, arglocs, resloc): - # _, rhsloc, sizeloc = arglocs - # size = sizeloc.value - # if size == 4: - # self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 0) # 0 means equal - # else: - # self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 0) + def emit_vec_float_eq(self, op, arglocs, resloc): + resloc, loc1, loc2, sizeloc = arglocs + size = sizeloc.value + if size == 4: + self.mc.xvcmpeqspx(resloc.value, loc1.value, loc2.value) + elif size == 8: + self.mc.xvcmpeqdpx(resloc.value, loc1.value, loc2.value) + else: + notimplemented("[ppc/assembler] float == for size %d" % size) - #def genop_vec_float_ne(self, op, arglocs, resloc): - # _, rhsloc, sizeloc = arglocs - # size = sizeloc.value - # # b(100) == 1 << 2 means not equal - # if size == 4: - # self.mc.CMPPS_xxi(resloc.value, rhsloc.value, 1 << 2) - # else: - # self.mc.CMPPD_xxi(resloc.value, rhsloc.value, 1 << 2) + def emit_vec_float_ne(self, op, arglocs, resloc): + self.emit_vec_float_eq(op, arglocs, resloc) + resloc, loc1, loc2, sizeloc = arglocs + self.mc.xxlandc(resloc.value, resloc.value, resloc.value) #def genop_vec_int_eq(self, op, arglocs, resloc): # _, rhsloc, sizeloc = arglocs @@ -629,8 +631,14 @@ prepare_vec_int_and = prepare_vec_arith prepare_vec_int_or = prepare_vec_arith prepare_vec_int_xor = prepare_vec_arith + + prepare_vec_float_eq = prepare_vec_arith + prepare_vec_float_ne = prepare_vec_float_eq + prepare_vec_int_eq = prepare_vec_float_eq + prepare_vec_int_ne = prepare_vec_float_eq del prepare_vec_arith + def _prepare_vec_store(self, op): descr = op.getdescr() assert isinstance(descr, ArrayDescr) @@ -661,8 +669,6 @@ resloc = self.force_allocate_vector_reg(op) return [resloc, loc0] - - def prepare_vec_arith_unary(self, op): a0 = op.getarg(0) loc0 = self.ensure_vector_reg(a0) @@ -674,19 +680,6 @@ prepare_vec_float_abs = prepare_vec_arith_unary del prepare_vec_arith_unary - #def prepare_vec_float_eq(self, op): - # assert isinstance(op, VectorOp) - # lhs = op.getarg(0) - # assert isinstance(lhs, VectorOp) - # args = op.getarglist() - # rhsloc = self.make_sure_var_in_reg(op.getarg(1), args) - # lhsloc = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # self.perform(op, [lhsloc, rhsloc, imm(lhs.bytesize)], lhsloc) - - #prepare_vec_float_ne = prepare_vec_float_eq - #prepare_vec_int_eq = prepare_vec_float_eq - #prepare_vec_int_ne = prepare_vec_float_eq - #def prepare_vec_pack_i(self, op): # # new_res = vec_pack_i(res, src, index, count) # assert isinstance(op, VectorOp) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -393,19 +393,13 @@ rop.SAVE_EXC_CLASS, rop.SAVE_EXCEPTION, rop.RESTORE_EXCEPTION, - rop.VEC_RAW_LOAD_I, - rop.VEC_RAW_LOAD_F, - rop.VEC_RAW_STORE, - rop.VEC_GETARRAYITEM_RAW_I, - rop.VEC_GETARRAYITEM_RAW_F, - rop.VEC_SETARRAYITEM_RAW, - rop.VEC_GETARRAYITEM_GC_I, - rop.VEC_GETARRAYITEM_GC_F, - rop.VEC_SETARRAYITEM_GC, + rop.VEC_LOAD_I, + rop.VEC_LOAD_F, rop.GC_LOAD_I, rop.GC_LOAD_R, rop.GC_LOAD_F, rop.GC_LOAD_INDEXED_R, + rop.VEC_STORE, rop.GC_STORE, rop.GC_STORE_INDEXED, rop.LOAD_FROM_GC_TABLE, diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py --- a/rpython/jit/metainterp/optimizeopt/schedule.py +++ b/rpython/jit/metainterp/optimizeopt/schedule.py @@ -1,3 +1,4 @@ +from rpython.jit.backend.llsupport.rewrite import cpu_simplify_scale from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT, ConstInt, ConstFloat, TargetToken) from rpython.jit.metainterp.resoperation import (rop, ResOperation, @@ -26,7 +27,8 @@ return fwd class SchedulerState(object): - def __init__(self, graph): + def __init__(self, cpu, graph): + self.cpu = cpu self.renamer = Renamer() self.graph = graph self.oplist = [] @@ -211,230 +213,16 @@ import pdb; pdb.set_trace() raise NotImplementedError(msg) -class TypeRestrict(object): - ANY_TYPE = '\x00' - ANY_SIZE = -1 - ANY_SIGN = -1 - ANY_COUNT = -1 - SIGNED = 1 - UNSIGNED = 0 - - def __init__(self, - type=ANY_TYPE, - bytesize=ANY_SIZE, - count=ANY_SIGN, - sign=ANY_COUNT): - self.type = type - self.bytesize = bytesize - self.sign = sign - self.count = count - - @always_inline - def any_size(self): - return self.bytesize == TypeRestrict.ANY_SIZE - - @always_inline - def any_count(self): - return self.count == TypeRestrict.ANY_COUNT - - def check(self, value): - vecinfo = forwarded_vecinfo(value) - assert vecinfo.datatype != '\x00' - if self.type != TypeRestrict.ANY_TYPE: - if self.type != vecinfo.datatype: - msg = "type mismatch %s != %s" % \ - (self.type, vecinfo.datatype) - failnbail_transformation(msg) - assert vecinfo.bytesize > 0 - if not self.any_size(): - if self.bytesize != vecinfo.bytesize: - msg = "bytesize mismatch %s != %s" % \ - (self.bytesize, vecinfo.bytesize) - failnbail_transformation(msg) - assert vecinfo.count > 0 - if self.count != TypeRestrict.ANY_COUNT: - if vecinfo.count < self.count: - msg = "count mismatch %s < %s" % \ - (self.count, vecinfo.count) - failnbail_transformation(msg) - if self.sign != TypeRestrict.ANY_SIGN: - if bool(self.sign) == vecinfo.sign: - msg = "sign mismatch %s < %s" % \ - (self.sign, vecinfo.sign) - failnbail_transformation(msg) - - def max_input_count(self, count): - """ How many """ - if self.count != TypeRestrict.ANY_COUNT: - return self.count - return count - -class OpRestrict(object): - def __init__(self, argument_restris): - self.argument_restrictions = argument_restris - - def check_operation(self, state, pack, op): - pass - - def crop_vector(self, op, newsize, size): - return newsize, size - - def must_crop_vector(self, op, index): - restrict = self.argument_restrictions[index] - vecinfo = forwarded_vecinfo(op.getarg(index)) - size = vecinfo.bytesize - newsize = self.crop_to_size(op, index) - return not restrict.any_size() and newsize != size - - @always_inline - def crop_to_size(self, op, index): - restrict = self.argument_restrictions[index] - return restrict.bytesize - - def opcount_filling_vector_register(self, op, vec_reg_size): - """ How many operations of that kind can one execute - with a machine instruction of register size X? - """ - if op.is_typecast(): - if op.casts_down(): - size = op.cast_input_bytesize(vec_reg_size) - return size // op.cast_from_bytesize() - else: - return vec_reg_size // op.cast_to_bytesize() - vecinfo = forwarded_vecinfo(op) - return vec_reg_size // vecinfo.bytesize - -class GuardRestrict(OpRestrict): - def opcount_filling_vector_register(self, op, vec_reg_size): - arg = op.getarg(0) - vecinfo = forwarded_vecinfo(arg) - return vec_reg_size // vecinfo.bytesize - -class LoadRestrict(OpRestrict): - def opcount_filling_vector_register(self, op, vec_reg_size): - assert rop.is_primitive_load(op.opnum) - descr = op.getdescr() - return vec_reg_size // descr.get_item_size_in_bytes() - -class StoreRestrict(OpRestrict): - def __init__(self, argument_restris): - self.argument_restrictions = argument_restris - - def must_crop_vector(self, op, index): - vecinfo = forwarded_vecinfo(op.getarg(index)) - bytesize = vecinfo.bytesize - return self.crop_to_size(op, index) != bytesize - - @always_inline - def crop_to_size(self, op, index): - # there is only one parameter that needs to be transformed! - descr = op.getdescr() - return descr.get_item_size_in_bytes() - - def opcount_filling_vector_register(self, op, vec_reg_size): - assert rop.is_primitive_store(op.opnum) - descr = op.getdescr() - return vec_reg_size // descr.get_item_size_in_bytes() - -class OpMatchSizeTypeFirst(OpRestrict): - def check_operation(self, state, pack, op): - i = 0 - infos = [forwarded_vecinfo(o) for o in op.getarglist()] - arg0 = op.getarg(i) - while arg0.is_constant() and i < op.numargs(): - i += 1 - arg0 = op.getarg(i) - vecinfo = forwarded_vecinfo(arg0) - bytesize = vecinfo.bytesize - datatype = vecinfo.datatype - - for arg in op.getarglist(): - if arg.is_constant(): - continue - curvecinfo = forwarded_vecinfo(arg) - if curvecinfo.bytesize != bytesize: - raise NotAVectorizeableLoop() - if curvecinfo.datatype != datatype: - raise NotAVectorizeableLoop() - -class trans(object): - - TR_ANY = TypeRestrict() - TR_ANY_FLOAT = TypeRestrict(FLOAT) - TR_ANY_INTEGER = TypeRestrict(INT) - TR_FLOAT_2 = TypeRestrict(FLOAT, 4, 2) - TR_DOUBLE_2 = TypeRestrict(FLOAT, 8, 2) - TR_INT32_2 = TypeRestrict(INT, 4, 2) - - OR_MSTF_I = OpMatchSizeTypeFirst([TR_ANY_INTEGER, TR_ANY_INTEGER]) - OR_MSTF_F = OpMatchSizeTypeFirst([TR_ANY_FLOAT, TR_ANY_FLOAT]) - STORE_RESTRICT = StoreRestrict([None, None, TR_ANY]) - LOAD_RESTRICT = LoadRestrict([]) - GUARD_RESTRICT = GuardRestrict([TR_ANY_INTEGER]) - - # note that the following definition is x86 arch specific - MAPPING = { - rop.VEC_INT_ADD: OR_MSTF_I, - rop.VEC_INT_SUB: OR_MSTF_I, - rop.VEC_INT_MUL: OR_MSTF_I, - rop.VEC_INT_AND: OR_MSTF_I, - rop.VEC_INT_OR: OR_MSTF_I, - rop.VEC_INT_XOR: OR_MSTF_I, - rop.VEC_INT_EQ: OR_MSTF_I, - rop.VEC_INT_NE: OR_MSTF_I, - - rop.VEC_FLOAT_ADD: OR_MSTF_F, - rop.VEC_FLOAT_SUB: OR_MSTF_F, - rop.VEC_FLOAT_MUL: OR_MSTF_F, - rop.VEC_FLOAT_TRUEDIV: OR_MSTF_F, - rop.VEC_FLOAT_ABS: OpRestrict([TR_ANY_FLOAT]), - rop.VEC_FLOAT_NEG: OpRestrict([TR_ANY_FLOAT]), - - rop.VEC_RAW_STORE: STORE_RESTRICT, - rop.VEC_SETARRAYITEM_RAW: STORE_RESTRICT, - rop.VEC_SETARRAYITEM_GC: STORE_RESTRICT, - - rop.VEC_RAW_LOAD_I: LOAD_RESTRICT, - rop.VEC_RAW_LOAD_F: LOAD_RESTRICT, - rop.VEC_GETARRAYITEM_RAW_I: LOAD_RESTRICT, - rop.VEC_GETARRAYITEM_RAW_F: LOAD_RESTRICT, - rop.VEC_GETARRAYITEM_GC_I: LOAD_RESTRICT, - rop.VEC_GETARRAYITEM_GC_F: LOAD_RESTRICT, - - rop.VEC_GUARD_TRUE: GUARD_RESTRICT, - rop.VEC_GUARD_FALSE: GUARD_RESTRICT, - - ## irregular - rop.VEC_INT_SIGNEXT: OpRestrict([TR_ANY_INTEGER]), - - rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpRestrict([TR_DOUBLE_2]), - # weird but the trace will store single floats in int boxes - rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpRestrict([TR_INT32_2]), - rop.VEC_CAST_FLOAT_TO_INT: OpRestrict([TR_DOUBLE_2]), - rop.VEC_CAST_INT_TO_FLOAT: OpRestrict([TR_INT32_2]), - - rop.VEC_FLOAT_EQ: OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]), - rop.VEC_FLOAT_NE: OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]), - rop.VEC_INT_IS_TRUE: OpRestrict([TR_ANY_INTEGER,TR_ANY_INTEGER]), - } - - @staticmethod - def get(op): - res = trans.MAPPING.get(op.vector, None) - if not res: - failnbail_transformation("could not get OpRestrict for " + str(op)) - return res - def turn_into_vector(state, pack): """ Turn a pack into a vector instruction """ check_if_pack_supported(state, pack) state.costmodel.record_pack_savings(pack, pack.numops()) left = pack.leftmost() - oprestrict = trans.get(left) + oprestrict = state.cpu.vector_ext.get_operation_restriction(left) if oprestrict is not None: oprestrict.check_operation(state, pack, left) args = left.getarglist_copy() - prepare_arguments(state, pack, args) + prepare_arguments(state, oprestrict, pack, args) vecop = VecOperation(left.vector, args, left, pack.numops(), left.getdescr()) for i,node in enumerate(pack.operations): @@ -449,7 +237,7 @@ state.oplist.append(vecop) assert vecop.count >= 1 -def prepare_arguments(state, pack, args): +def prepare_arguments(state, oprestrict, pack, args): # Transforming one argument to a vector box argument # The following cases can occur: # 1) argument is present in the box_to_vbox map. @@ -461,7 +249,6 @@ # a) expand vars/consts before the label and add as argument # b) expand vars created in the loop body # - oprestrict = trans.MAPPING.get(pack.leftmost().vector, None) if not oprestrict: return restrictions = oprestrict.argument_restrictions @@ -684,9 +471,8 @@ class VecScheduleState(SchedulerState): def __init__(self, graph, packset, cpu, costmodel): - SchedulerState.__init__(self, graph) + SchedulerState.__init__(self, cpu, graph) self.box_to_vbox = {} - self.cpu = cpu self.vec_reg_size = cpu.vector_register_size self.expanded_map = {} self.costmodel = costmodel diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py --- a/rpython/jit/metainterp/optimizeopt/vector.py +++ b/rpython/jit/metainterp/optimizeopt/vector.py @@ -132,7 +132,7 @@ # start = time.clock() opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, warmstate.vec_cost) - index_vars = opt.run_optimization(info, loop) + index_vars = opt.run_optimization(metainterp_sd, info, loop) gso = GuardStrengthenOpt(index_vars) gso.propagate_all_forward(info, loop, user_code) end = time.clock() @@ -222,7 +222,7 @@ self.smallest_type_bytes = 0 self.orig_label_args = None - def run_optimization(self, info, loop): + def run_optimization(self, metainterp_sd, info, loop): self.orig_label_args = loop.label.getarglist_copy() self.linear_find_smallest_type(loop) byte_count = self.smallest_type_bytes @@ -235,7 +235,7 @@ # find index guards and move to the earliest position graph = self.analyse_index_calculations(loop) if graph is not None: - state = SchedulerState(graph) + state = SchedulerState(metainterp_sd.cpu, graph) self.schedule(state) # reorder the trace # unroll diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1074,11 +1074,9 @@ '_RAW_LOAD_FIRST', 'GETARRAYITEM_GC/2d/rfi', - 'VEC_GETARRAYITEM_GC/2d/fi', 'GETARRAYITEM_RAW/2d/fi', - 'VEC_GETARRAYITEM_RAW/2d/fi', 'RAW_LOAD/2d/fi', - 'VEC_RAW_LOAD/2d/fi', + 'VEC_LOAD/2d/fi', '_RAW_LOAD_LAST', 'GETINTERIORFIELD_GC/2d/rfi', @@ -1111,11 +1109,9 @@ 'INCREMENT_DEBUG_COUNTER/1/n', '_RAW_STORE_FIRST', 'SETARRAYITEM_GC/3d/n', - 'VEC_SETARRAYITEM_GC/3d/n', 'SETARRAYITEM_RAW/3d/n', - 'VEC_SETARRAYITEM_RAW/3d/n', 'RAW_STORE/3d/n', - 'VEC_RAW_STORE/3d/n', + 'VEC_STORE/3d/n', '_RAW_STORE_LAST', 'SETINTERIORFIELD_GC/3d/n', 'SETINTERIORFIELD_RAW/3d/n', # right now, only used by tests @@ -1711,19 +1707,19 @@ rop.PTR_NE: rop.PTR_NE, } _opvector = { - rop.RAW_LOAD_I: rop.VEC_RAW_LOAD_I, - rop.RAW_LOAD_F: rop.VEC_RAW_LOAD_F, - rop.GETARRAYITEM_RAW_I: rop.VEC_GETARRAYITEM_RAW_I, - rop.GETARRAYITEM_RAW_F: rop.VEC_GETARRAYITEM_RAW_F, - rop.GETARRAYITEM_GC_I: rop.VEC_GETARRAYITEM_GC_I, - rop.GETARRAYITEM_GC_F: rop.VEC_GETARRAYITEM_GC_F, + rop.RAW_LOAD_I: rop.VEC_LOAD_I, + rop.RAW_LOAD_F: rop.VEC_LOAD_F, + rop.GETARRAYITEM_RAW_I: rop.VEC_LOAD_I, + rop.GETARRAYITEM_RAW_F: rop.VEC_LOAD_F, + rop.GETARRAYITEM_GC_I: rop.VEC_LOAD_I, + rop.GETARRAYITEM_GC_F: rop.VEC_LOAD_F, # note that there is no _PURE operation for vector operations. # reason: currently we do not care if it is pure or not! - rop.GETARRAYITEM_GC_PURE_I: rop.VEC_GETARRAYITEM_GC_I, - rop.GETARRAYITEM_GC_PURE_F: rop.VEC_GETARRAYITEM_GC_F, - rop.RAW_STORE: rop.VEC_RAW_STORE, - rop.SETARRAYITEM_RAW: rop.VEC_SETARRAYITEM_RAW, - rop.SETARRAYITEM_GC: rop.VEC_SETARRAYITEM_GC, + rop.GETARRAYITEM_GC_PURE_I: rop.VEC_LOAD_I, + rop.GETARRAYITEM_GC_PURE_F: rop.VEC_LOAD_F, + rop.RAW_STORE: rop.VEC_STORE, + rop.SETARRAYITEM_RAW: rop.VEC_STORE, + rop.SETARRAYITEM_GC: rop.VEC_STORE, rop.INT_ADD: rop.VEC_INT_ADD, rop.INT_SUB: rop.VEC_INT_SUB, diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -28,7 +28,11 @@ lltype.free(mem, flavor='raw') def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) \ + or (math.isnan(a) and math.isnan(b)) or \ + (math.isinf(a) and math.isinf(b) and \ + (a < 0.0 and b < 0.0) or \ + (a > 0.0 and b > 0.0)) class RawStorage(object): def __init__(self): @@ -84,6 +88,8 @@ type_system=self.type_system, vec=vec, vec_all=vec_all) + # FLOAT UNARY + def _vector_float_unary(self, func, type, data): func = always_inline(func) @@ -109,10 +115,7 @@ for i in range(l): c = raw_storage_getitem(type,vc,i*size) r = func(la[i]) - assert isclose(r, c) or (math.isnan(r) and math.isnan(c)) or \ - (math.isinf(r) and math.isinf(c) and \ - (r < 0.0 and c < 0.0) or \ - (r > 0.0 and c > 0.0)) + assert isclose(r, c) rawstorage.clear() @@ -125,15 +128,14 @@ test_vec_abs_float = \ vec_float_unary(lambda v: abs(v), rffi.DOUBLE) + test_vec_neg_float = \ + vec_float_unary(lambda v: -v, rffi.DOUBLE) + # FLOAT BINARY - @given(data=st.data()) - @pytest.mark.parametrize('func', [lambda a,b: a+b, - lambda a,b: a*b, lambda a,b: a-b]) - def test_vector_simple_float(self, func, data): + def _vector_simple_float(self, func, type, data): func = always_inline(func) - type = rffi.DOUBLE size = rffi.sizeof(rffi.DOUBLE) myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) def f(bytecount, va, vb, vc): @@ -159,15 +161,29 @@ self.meta_interp(f, [l*size, va, vb, vc]) for i in range(l): + import pdb; pdb.set_trace() c = raw_storage_getitem(type,vc,i*size) r = func(la[i], lb[i]) - assert isclose(r, c) or (math.isnan(r) and math.isnan(c)) or \ - (math.isinf(r) and math.isinf(c) and \ - (r < 0.0 and c < 0.0) or \ - (r > 0.0 and c > 0.0)) + assert isclose(r, c) rawstorage.clear() + def _vec_float_binary(test_func, func, type): + return pytest.mark.parametrize('func,type', [ + (func, type) + ])(given(data=st.data())(test_func)) + + vec_float_binary = functools.partial(_vec_float_binary, _vector_simple_float) + + test_vector_float_add = \ + vec_float_binary(lambda a,b: a+b, rffi.DOUBLE) + test_vector_float_sub = \ + vec_float_binary(lambda a,b: a-b, rffi.DOUBLE) + test_vector_float_mul = \ + vec_float_binary(lambda a,b: a*b, rffi.DOUBLE) + #test_vector_float_div = \ + # vec_float_binary(lambda a,b: a/b, rffi.DOUBLE) + def _vector_simple_int(self, func, type, data): func = always_inline(func) From pypy.commits at gmail.com Mon Jun 27 10:13:12 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 07:13:12 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <57713478.a758c20a.2a731.7587@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85399:6df99c485ecf Date: 2016-06-27 16:14 +0200 http://bitbucket.org/pypy/pypy/changeset/6df99c485ecf/ Log: in-progress diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -107,6 +107,7 @@ #else /* see revdb.c */ RPY_EXTERN void rpy_reverse_db_next_dead(void *); +RPY_EXTERN int rpy_reverse_db_fq_register(void *); #endif static void mem_boehm_ignore(char *msg, GC_word arg) @@ -121,7 +122,7 @@ GC_set_warn_proc(mem_boehm_ignore); } -void boehm_fq_callback(void *obj, void *rawfqueue) +static void boehm_fq_callback(void *obj, void *rawfqueue) { struct boehm_fq_s **fqueue = rawfqueue; struct boehm_fq_s *node = GC_malloc(sizeof(void *) * 2); @@ -132,6 +133,16 @@ *fqueue = node; } +void boehm_fq_register(struct boehm_fq_s **fqueue, void *obj) +{ +#ifdef RPY_REVERSE_DEBUGGER + /* this function returns 0 when recording, or 1 when replaying */ + if (rpy_reverse_db_fq_register(obj)) + return; +#endif + GC_REGISTER_FINALIZER(obj, boehm_fq_callback, fqueue, NULL, NULL); +} + void *boehm_fq_next_dead(struct boehm_fq_s **fqueue) { struct boehm_fq_s *node = *fqueue; diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -107,7 +107,7 @@ struct boehm_fq_s; RPY_EXTERN struct boehm_fq_s *boehm_fq_queues[]; RPY_EXTERN void (*boehm_fq_trigger[])(void); -RPY_EXTERN void boehm_fq_callback(void *, void *); +RPY_EXTERN void boehm_fq_register(struct boehm_fq_s **, void *); RPY_EXTERN void *boehm_fq_next_dead(struct boehm_fq_s **); #define OP_GC__DISABLE_FINALIZERS(r) boehm_gc_finalizer_lock++ @@ -115,8 +115,7 @@ boehm_gc_finalizer_notifier()) #define OP_BOEHM_FQ_REGISTER(tagindex, obj, r) \ - GC_REGISTER_FINALIZER(obj, boehm_fq_callback, \ - boehm_fq_queues + tagindex, NULL, NULL) + boehm_fq_register(boehm_fq_queues + tagindex, obj) #define OP_BOEHM_FQ_NEXT_DEAD(tagindex, r) \ r = boehm_fq_next_dead(boehm_fq_queues + tagindex) diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -1,4 +1,4 @@ -import sys, re +import sys, os, re import subprocess, socket import traceback from contextlib import contextmanager @@ -15,12 +15,15 @@ def __init__(self, revdb_log_filename, executable=None): with open(revdb_log_filename, 'rb') as f: header = f.readline() - fields = header.split('\t') + assert header.endswith('\n') + fields = header[:-1].split('\t') if len(fields) < 2 or fields[0] != 'RevDB:': raise ValueError("file %r is not a RevDB log" % ( revdb_log_filename,)) if executable is None: executable = fields[1] + if not os.path.isfile(executable): + raise ValueError("executable %r not found" % (executable,)) self.pgroup = ReplayProcessGroup(executable, revdb_log_filename) self.print_extra_pending_info = None diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "structdef.h" #include "forwarddecl.h" @@ -73,6 +74,11 @@ void rpy_reverse_db_teardown(void) { uint64_t stop_points; + if (RPY_RDB_REPLAY) { + /* hack: prevents RPY_REVDB_EMIT() from calling + rpy_reverse_db_fetch(), which has nothing more to fetch now */ + rpy_revdb.buf_limit += 1; + } RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; , uint64_t _e, stop_points); @@ -173,8 +179,9 @@ { ssize_t content_size = current_packet_size(); if (content_size != 0) { + char *p = rpy_rev_buffer; assert(0 < content_size && content_size <= 32767); - *(int16_t *)rpy_rev_buffer = content_size; + *(int16_t *)p = content_size; flush_buffer(); } } @@ -204,6 +211,8 @@ Then, call boehm_fq_trigger(), which calls finalizer_trigger(). */ int i; + char *p = rpy_rev_buffer; + rpy_reverse_db_flush(); GC_invoke_finalizers(); @@ -219,7 +228,7 @@ "record_stop_point emitted unexpectedly to the rdb log\n"); exit(1); } - *(int16_t *)rpy_rev_buffer = ASYNC_FINALIZER_TRIGGER; + *(int16_t *)p = ASYNC_FINALIZER_TRIGGER; memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t)); rpy_revdb.buf_p += sizeof(uint64_t); flush_buffer(); @@ -294,7 +303,8 @@ exit(1); } if (pwrite(rpy_rev_fileno, &new, 1, offset) != 1) { - fprintf(stderr, "can't patch log position %lld\n", offset); + fprintf(stderr, "can't patch log position %lld\n", + (long long)offset); exit(1); } } @@ -446,6 +456,7 @@ static uint64_t stopped_time; static uint64_t stopped_uid; static uint64_t total_stop_points; +static uint64_t finalizer_trigger_saved_break; static jmp_buf jmp_buf_cancel_execution; static void (*pending_after_forward)(void); static RPyString *empty_string; @@ -453,6 +464,7 @@ static int last_recorded_breakpoint_num; static char breakpoint_mode; static uint64_t *future_ids, *future_next_id; +static void *finalizer_tree; static void attach_gdb(void) { @@ -606,6 +618,11 @@ /* ignore the SIGCHLD signals so that child processes don't become zombies */ signal(SIGCHLD, SIG_IGN); + + /* initiate the read, which is always at least one byte ahead of + RPY_REVDB_EMIT() in order to detect the ASYNC_* operations + early enough. */ + rpy_reverse_db_fetch(__FILE__, __LINE__); } static void fetch_more(ssize_t keep, ssize_t expected_size) @@ -622,15 +639,19 @@ } RPY_EXTERN -char *rpy_reverse_db_fetch(const char *file, int line) +void rpy_reverse_db_fetch(const char *file, int line) { if (!flag_io_disabled) { ssize_t keep; ssize_t full_packet_size; + int16_t header; + if (rpy_revdb.buf_limit != rpy_revdb.buf_p) { fprintf(stderr, "bad log format: incomplete packet\n"); exit(1); } + + fetch_more_if_needed: keep = rpy_revdb.buf_readend - rpy_revdb.buf_p; assert(keep >= 0); @@ -639,13 +660,41 @@ fetch_more(keep, sizeof(int16_t)); keep = rpy_revdb.buf_readend - rpy_rev_buffer; } - full_packet_size = sizeof(int16_t) + *(int16_t *)rpy_revdb.buf_p; - if (keep < full_packet_size) { + header = *(int16_t *)rpy_revdb.buf_p; + if (header < 0) { + int64_t bp; + + switch (header) { + + case ASYNC_FINALIZER_TRIGGER: + if (finalizer_trigger_saved_break != 0) { + fprintf(stderr, "unexpected multiple " + "ASYNC_FINALIZER_TRIGGER\n"); + exit(1); + } + full_packet_size = sizeof(int16_t) + sizeof(int64_t); + if (keep < full_packet_size) + fetch_more(keep, full_packet_size); + memcpy(&bp, rpy_revdb.buf_p + sizeof(int16_t), sizeof(int64_t)); + rpy_revdb.buf_p += full_packet_size; + if (bp <= rpy_revdb.stop_point_seen) { + fprintf(stderr, "invalid finalizer break point\n"); + exit(1); + } + finalizer_trigger_saved_break = rpy_revdb.stop_point_break; + rpy_revdb.stop_point_break = bp; + goto fetch_more_if_needed; + + default: + fprintf(stderr, "bad packet header %d", (int)header); + exit(1); + } + } + full_packet_size = sizeof(int16_t) + header; + if (keep < full_packet_size) fetch_more(keep, full_packet_size); - } rpy_revdb.buf_limit = rpy_revdb.buf_p + full_packet_size; rpy_revdb.buf_p += sizeof(int16_t); - return rpy_revdb.buf_p; } else { /* this is called when we are in execute_rpy_command(): we are @@ -672,8 +721,8 @@ disabled_exc[1] = pypy_g_ExcData.ed_exc_value; pypy_g_ExcData.ed_exc_type = NULL; pypy_g_ExcData.ed_exc_value = NULL; - rpy_revdb.buf_p = NULL; - rpy_revdb.buf_limit = NULL; + rpy_revdb.buf_p = rpy_rev_buffer; /* anything readable */ + rpy_revdb.buf_limit = rpy_rev_buffer; /* same as buf_p */ flag_io_disabled = 1; } @@ -724,7 +773,7 @@ (long long)stop_points); exit(1); } - if (rpy_revdb.buf_p != rpy_revdb.buf_limit || + if (rpy_revdb.buf_p != rpy_revdb.buf_limit - 1 || read(rpy_rev_fileno, dummy, 1) > 0) { fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); exit(1); @@ -900,6 +949,10 @@ static void replay_stop_point(void) { + if (finalizer_trigger_saved_break != 0) { + abort(); + } + while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { save_state(); breakpoint_mode = 0; @@ -911,8 +964,6 @@ } else { rpy_revdb_command_t cmd; - if (((int64_t)stopped_uid) < 0) - attach_gdb(); write_answer(ANSWER_READY, stopped_time, stopped_uid, 0); read_sock(&cmd, sizeof(cmd)); @@ -1028,5 +1079,32 @@ return uid; } +static int _ftree_compare(const void *obj1, const void *obj2) +{ + const struct pypy_header0 *h1 = obj1; + const struct pypy_header0 *h2 = obj2; + if (h1->h_uid < h2->h_uid) + return -1; + if (h1->h_uid == h2->h_uid) + return 0; + else + return 1; +} + +RPY_EXTERN +int rpy_reverse_db_fq_register(void *obj) +{ + if (!RPY_RDB_REPLAY) { + return 0; /* recording */ + } + else { + void *added = tsearch(obj, &finalizer_tree, _ftree_compare); + if (added != obj) { + fprintf(stderr, "tsearch: duplicate object\n"); + exit(1); + } + return 1; /* replaying */ + } +} /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -52,15 +52,13 @@ decl_e; \ char *_src = rpy_revdb.buf_p; \ char *_end1 = _src + sizeof(_e); \ - if (_end1 > rpy_revdb.buf_limit) { \ - _src = rpy_reverse_db_fetch(__FILE__, __LINE__); \ - _end1 = _src + sizeof(_e); \ - } \ + memcpy(&_e, _src, sizeof(_e)); \ rpy_revdb.buf_p = _end1; \ - memcpy(&_e, _src, sizeof(_e)); \ _RPY_REVDB_PRINT((stderr, "%s:%d: read %0*llx\n", \ __FILE__, __LINE__, \ 2 * sizeof(_e), (unsigned long long)_e)); \ + if (_end1 >= rpy_revdb.buf_limit) \ + rpy_reverse_db_fetch(__FILE__, __LINE__); \ variable = _e; \ } @@ -116,7 +114,7 @@ rpy_reverse_db_call_destructor(obj) RPY_EXTERN void rpy_reverse_db_flush(void); -RPY_EXTERN char *rpy_reverse_db_fetch(const char *file, int line); +RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line); RPY_EXTERN void rpy_reverse_db_stop_point(void); RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2, int64_t arg3, RPyString *extra); @@ -129,6 +127,7 @@ RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target); RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref); //RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); +RPY_EXTERN int rpy_reverse_db_fq_register(void *obj); RPY_EXTERN void rpy_reverse_db_next_dead(void *result); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -84,10 +84,11 @@ return self.cur == len(self.buffer) -def compile(self, entry_point, argtypes, backendopt=True, +def compile(self, entry_point, backendopt=True, withsmallfuncsets=None): t = Translation(entry_point, None, gc="boehm") self.t = t + t.set_backend_extra_options(c_debug_defines=True) t.config.translation.reverse_debugger = True t.config.translation.lldebug0 = True if withsmallfuncsets is not None: @@ -127,7 +128,7 @@ def main(argv): print argv[1:] return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) assert self.run('abc d') == '[abc, d]\n' rdb = self.fetch_rdb([self.exename, 'abc', 'd']) # write() call @@ -143,7 +144,7 @@ objectmodel.compute_identity_hash(argv), objectmodel.compute_identity_hash(argv)] return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') match = re.match(r'\[(-?\d+), \1, \1]\n', out) assert match @@ -167,7 +168,7 @@ def main(argv): print lst[len(argv) & 1].x return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') assert out == '42\n' rdb = self.fetch_rdb([self.exename, 'Xx']) @@ -190,7 +191,7 @@ def main(argv): print lst[len(argv) & 1].x return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') assert out == '41\n' rdb = self.fetch_rdb([self.exename, 'Xx']) @@ -220,7 +221,7 @@ x = f3 # now can be f1 or f2 or f3 print x() return 9 - self.compile(main, [], backendopt=False, withsmallfuncsets=limit) + self.compile(main, backendopt=False, withsmallfuncsets=limit) for input, expected_output in [ ('2 3', '111\n'), ('2 3 4', '222\n'), @@ -265,7 +266,7 @@ for x in lst: print revdb.get_unique_id(x) return 9 - compile(cls, main, [], backendopt=False) + compile(cls, main, backendopt=False) assert run(cls, 'abc d ef') == ('abc\nd\nef\n' '3\n0\n12\n15\n17\n') rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef']) @@ -324,8 +325,7 @@ debug_print('<<<', cmd.c_cmd, cmd.c_arg1, cmd.c_arg2, cmd.c_arg3, extra, '>>>') if extra == 'oops': - for i in range(1000): - print 42 # I/O not permitted + print 42 # I/O not permitted if extra == 'raise-and-catch': try: g(extra) @@ -373,7 +373,7 @@ revdb.stop_point() print op return 9 - compile(cls, main, [], backendopt=False) + compile(cls, main, backendopt=False) assert run(cls, 'abc d ef') == 'abc\nd\nef\n' def test_run_blip(self): diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py --- a/rpython/translator/revdb/test/test_process.py +++ b/rpython/translator/revdb/test/test_process.py @@ -37,7 +37,7 @@ revdb.stop_point() print op return 9 - compile(cls, main, [], backendopt=False) + compile(cls, main, backendopt=False) assert run(cls, 'abc d ef g h i j k l m') == ( 'abc\nd\nef\ng\nh\ni\nj\nk\nl\nm\n') diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -26,6 +26,52 @@ ASYNC_FINALIZER_TRIGGER = 0xff46 - 2**16 +def get_finalizer_queue_main(): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.translator.tool.cbuild import ExternalCompilationInfo + eci = ExternalCompilationInfo( + pre_include_bits=["#define foobar(x) x\n"]) + foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, + compilation_info=eci) + class Glob: + pass + glob = Glob() + class X: + pass + class MyFinalizerQueue(rgc.FinalizerQueue): + Class = X + def finalizer_trigger(self): + glob.ping = True + fq = MyFinalizerQueue() + # + def main(argv): + glob.ping = False + lst1 = [X() for i in range(256)] + lst = [X() for i in range(3000)] + for i, x in enumerate(lst): + x.baz = i + fq.register_finalizer(x) + for i in range(3000): + lst[i] = None + if i % 300 == 150: + rgc.collect() + revdb.stop_point() + j = i + glob.ping * 1000000 + assert foobar(j) == j + if glob.ping: + glob.ping = False + total = 0 + while True: + x = fq.next_dead() + if x is None: + break + total = intmask(total * 3 + x.baz) + assert foobar(total) == total + keepalive_until_here(lst1) + return 9 + return main + + class TestRecording(BaseRecordingTests): def test_weakref_create(self): @@ -39,7 +85,7 @@ glob.r2 = weakref.ref(X()) glob.r3 = weakref.ref(X()) return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) # find the extra WEAKREF_DEAD @@ -63,7 +109,7 @@ assert r1() is x1 # (*) assert r2() is x2 # (*) return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) # find the 2 + 16998 first WEAKREF_xxx (all "(*)" but the last two) @@ -87,56 +133,15 @@ rgc.collect() revdb.stop_point() return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) x = rdb.next('q'); assert x == 3000 # number of stop points assert rdb.done() def test_finalizer_queue(self): - from rpython.rtyper.lltypesystem import lltype, rffi - from rpython.translator.tool.cbuild import ExternalCompilationInfo - eci = ExternalCompilationInfo( - pre_include_bits=["#define foobar(x) x\n"]) - foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, - compilation_info=eci) - class Glob: - pass - glob = Glob() - class X: - pass - class MyFinalizerQueue(rgc.FinalizerQueue): - Class = X - def finalizer_trigger(self): - glob.ping = True - fq = MyFinalizerQueue() - # - def main(argv): - glob.ping = False - lst1 = [X() for i in range(256)] - lst = [X() for i in range(3000)] - for i, x in enumerate(lst): - x.baz = i - fq.register_finalizer(x) - for i in range(3000): - lst[i] = None - if i % 300 == 150: - rgc.collect() - revdb.stop_point() - j = i + glob.ping * 1000000 - assert foobar(j) == j - if glob.ping: - glob.ping = False - total = 0 - while True: - x = fq.next_dead() - if x is None: - break - total = intmask(total * 3 + x.baz) - assert foobar(total) == total - keepalive_until_here(lst1) - return 9 - self.compile(main, [], backendopt=False) + main = get_finalizer_queue_main() + self.compile(main, backendopt=False) out = self.run('Xx') rdb = self.fetch_rdb([self.exename, 'Xx']) uid_seen = set() @@ -196,7 +201,7 @@ assert foobar(x) == x print x return 9 - self.compile(main, [], backendopt=False) + self.compile(main, backendopt=False) out = self.run('Xx') assert 1500 < int(out) <= 3000 rdb = self.fetch_rdb([self.exename, 'Xx']) @@ -212,7 +217,7 @@ assert rdb.done() -class TestReplaying(InteractiveTests): +class TestReplayingWeakref(InteractiveTests): expected_stop_points = 1 def setup_class(cls): @@ -255,7 +260,7 @@ keepalive_until_here(keepalive) revdb.stop_point() return 9 - compile(cls, main, [], backendopt=False) + compile(cls, main, backendopt=False) output = run(cls, '') lines = output.splitlines() assert lines[-1].startswith('prebuilt') and lines[-1].endswith( @@ -268,3 +273,18 @@ # the asserts are replayed; if we get here it means they passed again child.send(Message(CMD_FORWARD, 1)) child.expect(ANSWER_AT_END) + + +class TestReplayingFinalizerQueue(InteractiveTests): + expected_stop_points = 3000 + + def setup_class(cls): + from rpython.translator.revdb.test.test_basic import compile, run + main = get_finalizer_queue_main() + compile(cls, main, backendopt=False) + run(cls, '') + + def test_replaying_finalizer_queue(self): + child = self.replay() + child.send(Message(CMD_FORWARD, 3001)) + child.expect(ANSWER_AT_END) From pypy.commits at gmail.com Mon Jun 27 10:52:43 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 07:52:43 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Reloading finalizers Message-ID: <57713dbb.0ed11c0a.1f111.ffffd92e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85400:ec1166e69eea Date: 2016-06-27 16:53 +0200 http://bitbucket.org/pypy/pypy/changeset/ec1166e69eea/ Log: Reloading finalizers diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -106,7 +106,7 @@ } #else /* see revdb.c */ -RPY_EXTERN void rpy_reverse_db_next_dead(void *); +RPY_EXTERN void *rpy_reverse_db_next_dead(void *); RPY_EXTERN int rpy_reverse_db_fq_register(void *); #endif @@ -154,7 +154,7 @@ else result = NULL; #ifdef RPY_REVERSE_DEBUGGER - rpy_reverse_db_next_dead(result); + result = rpy_reverse_db_next_dead(result); #endif return result; } diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -204,6 +204,13 @@ rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1; } +static void fq_trigger(void) +{ + int i = 0; + while (boehm_fq_trigger[i]) + boehm_fq_trigger[i++](); +} + static void record_stop_point(void) { /* Invoke the finalizers now. This will call boehm_fq_callback(), @@ -216,9 +223,7 @@ rpy_reverse_db_flush(); GC_invoke_finalizers(); - i = 0; - while (boehm_fq_trigger[i]) - boehm_fq_trigger[i++](); + fq_trigger(); /* This should all be done without emitting anything to the rdb log. We check that, and emit just a ASYNC_FINALIZER_TRIGGER. @@ -235,13 +240,6 @@ } RPY_EXTERN -void rpy_reverse_db_next_dead(void *result) -{ - int64_t uid = result ? ((struct pypy_header0 *)result)->h_uid : -1; - RPY_REVDB_EMIT(/* nothing */, int64_t _e, uid); -} - -RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj) { /* Boehm only */ @@ -950,7 +948,9 @@ static void replay_stop_point(void) { if (finalizer_trigger_saved_break != 0) { - abort(); + rpy_revdb.stop_point_break = finalizer_trigger_saved_break; + finalizer_trigger_saved_break = 0; + fq_trigger(); } while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { @@ -1098,13 +1098,46 @@ return 0; /* recording */ } else { - void *added = tsearch(obj, &finalizer_tree, _ftree_compare); - if (added != obj) { - fprintf(stderr, "tsearch: duplicate object\n"); + /* add the object into the finalizer_tree, keyed by the h_uid */ + void **item = tsearch(obj, &finalizer_tree, _ftree_compare); + if (item == NULL) { + fprintf(stderr, "fq_register: out of memory\n"); + exit(1); + } + if (*item != obj) { + fprintf(stderr, "fq_register: duplicate object\n"); exit(1); } return 1; /* replaying */ } } +RPY_EXTERN +void *rpy_reverse_db_next_dead(void *result) +{ + int64_t uid; + RPY_REVDB_EMIT(uid = result ? ((struct pypy_header0 *)result)->h_uid : -1;, + int64_t _e, uid); + if (RPY_RDB_REPLAY) { + if (uid == -1) { + result = NULL; + } + else { + /* fetch and remove the object from the finalizer_tree */ + void **item; + struct pypy_header0 dummy; + dummy.h_uid = uid; + item = tfind(&dummy, &finalizer_tree, _ftree_compare); + if (item == NULL) { + fprintf(stderr, "next_dead: object not found\n"); + exit(1); + } + result = *item; + assert(((struct pypy_header0 *)result)->h_uid == uid); + tdelete(result, &finalizer_tree, _ftree_compare); + } + } + return result; +} + /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -128,6 +128,6 @@ RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref); //RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); RPY_EXTERN int rpy_reverse_db_fq_register(void *obj); -RPY_EXTERN void rpy_reverse_db_next_dead(void *result); +RPY_EXTERN void *rpy_reverse_db_next_dead(void *result); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -1,4 +1,4 @@ -import weakref +import py, weakref from rpython.rlib import revdb, rgc from rpython.rlib.debug import debug_print from rpython.rlib.objectmodel import keepalive_until_here From pypy.commits at gmail.com Mon Jun 27 11:39:32 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 27 Jun 2016 08:39:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement tuple_ and set_unpack opcodes, create unpack_helper method Message-ID: <577148b4.4f941c0a.a0ede.0a97@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85401:f2c1dacc1078 Date: 2016-06-27 17:38 +0200 http://bitbucket.org/pypy/pypy/changeset/f2c1dacc1078/ Log: Implement tuple_ and set_unpack opcodes, create unpack_helper method diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1330,42 +1330,27 @@ self.space.call_method(w_set, 'add', w_item) self.pushvalue(w_set) - def BUILD_SET_UNPACK(self, itemcount, next_instr): - self.BUILD_SET(itemcount, next_instr) - #w_sum = self.space.newset() - #for i in range(itemcount, 0, -1): - # w_item = self.popvalue() - # #self.space.peek(i) - # self.space.call_method(w_sum, 'update', w_item) - ##while itemcount != 0: - ## self.popvalue() - ## itemcount -= 1 - #self.pushvalue(w_sum) - - def BUILD_TUPLE_UNPACK(self, itemcount, next_instr): - self.BUILD_TUPLE(itemcount, next_instr) - #w_sum = self.space.newtuple() - #for i in range(itemcount, 0, -1): - # w_item = self.popvalue() - # #self.space.peek(i) - # self.space.call_method(w_sum, 'update', w_item) - ##while itemcount != 0: - ## self.popvalue() - ## itemcount -= 1 - #self.pushvalue(w_sum) - - def BUILD_LIST_UNPACK(self, itemcount, next_instr): + def unpack_helper(self, itemcount, next_instr): w_sum = [] for i in range(itemcount, 0, -1): - #for i in range(0, itemcount): - #w_item = self.popvalue() w_item = self.peekvalue(i-1) items = self.space.fixedview(w_item) w_sum.extend(items) - #w_sum.append(w_item) while itemcount != 0: self.popvalue() itemcount -= 1 + return w_sum + + def BUILD_SET_UNPACK(self, itemcount, next_instr): + w_sum = self.unpack_helper(itemcount, next_instr) + self.pushvalue(self.space.newset(w_sum)) + + def BUILD_TUPLE_UNPACK(self, itemcount, next_instr): + w_sum = self.unpack_helper(itemcount, next_instr) + self.pushvalue(self.space.newtuple(w_sum)) + + def BUILD_LIST_UNPACK(self, itemcount, next_instr): + w_sum = self.unpack_helper(itemcount, next_instr) self.pushvalue(self.space.newlist(w_sum)) #TODO From pypy.commits at gmail.com Mon Jun 27 11:51:19 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 27 Jun 2016 08:51:19 -0700 (PDT) Subject: [pypy-commit] pypy PyTuple_Type-subclass: close branch to be merged Message-ID: <57714b77.e153c20a.2ab5b.3ade@mx.google.com> Author: Matti Picus Branch: PyTuple_Type-subclass Changeset: r85402:f0c4c02e8250 Date: 2016-06-27 18:40 +0300 http://bitbucket.org/pypy/pypy/changeset/f0c4c02e8250/ Log: close branch to be merged From pypy.commits at gmail.com Mon Jun 27 11:51:21 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 27 Jun 2016 08:51:21 -0700 (PDT) Subject: [pypy-commit] pypy default: merge PyTuple_Type-subclass which changes the model of PyTupleObject and allows subclassing it Message-ID: <57714b79.c4cb1c0a.4ebc4.1495@mx.google.com> Author: Matti Picus Branch: Changeset: r85403:9bf5c0afe0d0 Date: 2016-06-27 18:41 +0300 http://bitbucket.org/pypy/pypy/changeset/9bf5c0afe0d0/ Log: merge PyTuple_Type-subclass which changes the model of PyTupleObject and allows subclassing it diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -26,54 +26,10 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -120,8 +120,8 @@ def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -44,8 +44,8 @@ Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -1,6 +1,20 @@ #include "Python.h" #include "structmember.h" +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong + #define PyThing_FromStringAndSize PyUnicode_FromStringAndSize + #define PyThing_FromString PyUnicode_FromString + # defin PyThing_Check PyUnicode_Check + #define _PyThing_AsString _PyUnicode_AsString +#else + #define PyThing_FromStringAndSize PyString_FromStringAndSize + #define PyThing_FromString PyString_FromString + #define PyThing_Check PyString_Check + #define _PyThing_AsString PyString_AsString +#endif + typedef struct { PyObject_HEAD int foo; /* the context holder */ @@ -88,7 +102,7 @@ static PyObject * foo_get_name(PyObject *self, void *closure) { - return PyString_FromStringAndSize("Foo Example", 11); + return PyThing_FromStringAndSize("Foo Example", 11); } static PyObject * @@ -114,7 +128,7 @@ { PyObject *format; - format = PyString_FromString(""); + format = PyThing_FromString(""); if (format == NULL) return NULL; return format; } @@ -130,11 +144,11 @@ foo_setattro(fooobject *self, PyObject *name, PyObject *value) { char *name_str; - if (!PyString_Check(name)) { + if (!PyThing_Check(name)) { PyErr_SetObject(PyExc_AttributeError, name); return -1; } - name_str = PyString_AsString(name); + name_str = _PyThing_AsString(name); if (strcmp(name_str, "set_foo") == 0) { long v = PyInt_AsLong(value); @@ -620,20 +634,65 @@ (destructor)custom_dealloc, /*tp_dealloc*/ }; +static PyTypeObject TupleLike = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.TupleLike", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; + + static PyObject *size_of_instances(PyObject *self, PyObject *t) { return PyInt_FromLong(((PyTypeObject *)t)->tp_basicsize); } + +static PyObject * is_TupleLike(PyObject *self, PyObject * t) +{ + int tf = t->ob_type == &TupleLike; + if (t->ob_type->tp_itemsize == 0) + return PyInt_FromLong(-1); + return PyInt_FromLong(tf); +} + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, {"newCustom", (PyCFunction)newCustom, METH_NOARGS, NULL}, {"size_of_instances", (PyCFunction)size_of_instances, METH_O, NULL}, + {"is_TupleLike", (PyCFunction)is_TupleLike, METH_O, NULL}, {NULL, NULL} /* Sentinel */ }; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "foo", + "Module Doc", + -1, + foo_functions, + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_foo(void) + +#else + +#define INITERROR return /* Initialize this module. */ #ifdef __GNUC__ @@ -644,8 +703,16 @@ PyMODINIT_FUNC initfoo(void) +#endif { - PyObject *m, *d; + PyObject *d; +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("foo", foo_functions); +#endif + if (module == NULL) + INITERROR; footype.tp_new = PyType_GenericNew; @@ -654,52 +721,59 @@ MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype2) < 0) - return; + INITERROR; if (PyType_Ready(&MetaType) < 0) - return; + INITERROR; if (PyType_Ready(&InitErrType) < 0) - return; + INITERROR; if (PyType_Ready(&SimplePropertyType) < 0) - return; + INITERROR; SimplePropertyType.tp_new = PyType_GenericNew; InitErrType.tp_new = PyType_GenericNew; CustomType.ob_type = &MetaType; if (PyType_Ready(&CustomType) < 0) - return; + INITERROR; UnicodeSubtype3.tp_flags = Py_TPFLAGS_DEFAULT; UnicodeSubtype3.tp_base = &UnicodeSubtype; UnicodeSubtype3.tp_bases = Py_BuildValue("(OO)", &UnicodeSubtype, &CustomType); if (PyType_Ready(&UnicodeSubtype3) < 0) + INITERROR; + + TupleLike.tp_base = &PyTuple_Type; + if (PyType_Ready(&TupleLike) < 0) return; - m = Py_InitModule("foo", foo_functions); - if (m == NULL) - return; - d = PyModule_GetDict(m); + + d = PyModule_GetDict(module); if (d == NULL) - return; + INITERROR; if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype3", (PyObject *) &UnicodeSubtype3) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "InitErrType", (PyObject *) &InitErrType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Property", (PyObject *) &SimplePropertyType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Custom", (PyObject *) &CustomType) < 0) - return; + INITERROR; + if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) + INITERROR; +#if PY_MAJOR_VERSION >=3 + return module; +#endif } diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -51,7 +51,7 @@ api._PyTuple_Resize(ar, 10) assert api.PyTuple_Size(ar[0]) == 10 for i in range(3, 10): - rffi.cast(PyTupleObject, py_tuple).c_ob_item[i] = make_ref( + rffi.cast(PyTupleObject, ar[0]).c_ob_item[i] = make_ref( space, space.wrap(42 + i)) w_tuple = from_ref(space, ar[0]) assert space.int_w(space.len(w_tuple)) == 10 @@ -151,3 +151,8 @@ """), ]) module.run() + + def test_tuple_subclass(self): + module = self.import_module(name='foo') + a = module.TupleLike([1, 2, 3]) + assert module.is_TupleLike(a) == 1 diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -2,10 +2,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.debug import fatalerror_notb from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, + build_type_checkers, PyVarObjectFields, cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - make_ref, from_ref, decref, + make_ref, from_ref, decref, incref, pyobj_has_w_obj, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -29,8 +29,8 @@ PyTupleObjectStruct = lltype.ForwardReference() PyTupleObject = lltype.Ptr(PyTupleObjectStruct) ObjectItems = rffi.CArray(PyObject) -PyTupleObjectFields = PyObjectFields + \ - (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +PyTupleObjectFields = PyVarObjectFields + \ + (("ob_item", ObjectItems),) cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) @bootstrap_function @@ -56,14 +56,12 @@ tuple_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_tuple.layout.typedef) - py_obj = typedescr.allocate(space, space.w_tuple) + py_obj = typedescr.allocate(space, space.w_tuple, length) py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_ob_item = lltype.malloc(ObjectItems, length, - flavor='raw', zero=True, - add_memory_pressure=True) - py_tup.c_ob_size = length - return py_tup + p = py_tup.c_ob_item + for i in range(py_tup.c_ob_size): + p[i] = lltype.nullptr(PyObject.TO) + return py_obj def tuple_attach(space, py_obj, w_obj): """ @@ -71,23 +69,24 @@ buffer must not be modified. """ items_w = space.fixedview(w_obj) - l = len(items_w) - p = lltype.malloc(ObjectItems, l, flavor='raw', - add_memory_pressure=True) + py_tup = rffi.cast(PyTupleObject, py_obj) + length = len(items_w) + if py_tup.c_ob_size < length: + raise oefmt(space.w_ValueError, + "tuple_attach called on object with ob_size %d but trying to store %d", + py_tup.c_ob_size, length) i = 0 try: - while i < l: - p[i] = make_ref(space, items_w[i]) + while i < length: + py_tup.c_ob_item[i] = make_ref(space, items_w[i]) i += 1 except: while i > 0: i -= 1 - decref(space, p[i]) - lltype.free(p, flavor='raw') + ob = py_tup.c_ob_item[i] + py_tup.c_ob_item[i] = lltype.nullptr(PyObject.TO) + decref(space, ob) raise - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_ob_size = l - py_tup.c_ob_item = p def tuple_realize(space, py_obj): """ @@ -108,7 +107,9 @@ "converting a PyTupleObject into a W_TupleObject, " "but found NULLs as items") items_w[i] = w_item - w_obj = space.newtuple(items_w) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_TupleObject, w_type) + w_obj.__init__(items_w) track_reference(space, py_obj, w_obj) return w_obj @@ -118,18 +119,16 @@ """ py_tup = rffi.cast(PyTupleObject, py_obj) p = py_tup.c_ob_item - if p: - for i in range(py_tup.c_ob_size): - decref(space, p[i]) - lltype.free(p, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + for i in range(py_tup.c_ob_size): + decref(space, p[i]) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @cpython_api([Py_ssize_t], PyObject, result_is_ll=True) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return new_empty_tuple(space, size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PyTuple_SetItem(space, ref, index, py_obj): @@ -185,25 +184,25 @@ ref = p_ref[0] if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - ref = rffi.cast(PyTupleObject, ref) - oldsize = ref.c_ob_size - oldp = ref.c_ob_item - newp = lltype.malloc(ObjectItems, newsize, zero=True, flavor='raw', - add_memory_pressure=True) + oldref = rffi.cast(PyTupleObject, ref) + oldsize = oldref.c_ob_size + p_ref[0] = new_empty_tuple(space, newsize) + newref = rffi.cast(PyTupleObject, p_ref[0]) try: if oldsize < newsize: to_cp = oldsize else: to_cp = newsize for i in range(to_cp): - newp[i] = oldp[i] + ob = oldref.c_ob_item[i] + incref(space, ob) + newref.c_ob_item[i] = ob except: - lltype.free(newp, flavor='raw') + decref(space, p_ref[0]) + p_ref[0] = lltype.nullptr(PyObject.TO) raise - ref.c_ob_item = newp - ref.c_ob_size = newsize - lltype.free(oldp, flavor='raw') - # in this version, p_ref[0] never needs to be updated + finally: + decref(space, ref) return 0 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -371,6 +371,8 @@ # (minimally, if tp_basicsize is zero we copy it from the base) if not pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize + if pto.c_tp_itemsize < base_pto.c_tp_itemsize: + pto.c_tp_itemsize = base_pto.c_tp_itemsize flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) @@ -597,7 +599,7 @@ @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): - from pypy.module.cpyext.object import PyObject_dealloc + from pypy.module.cpyext.object import _dealloc obj_pto = rffi.cast(PyTypeObjectPtr, obj) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) @@ -608,7 +610,7 @@ heaptype = rffi.cast(PyHeapTypeObject, obj) Py_DecRef(space, heaptype.c_ht_name) Py_DecRef(space, base_pyo) - PyObject_dealloc(space, obj) + _dealloc(space, obj) def type_alloc(space, w_metatype, itemsize=0): @@ -659,6 +661,8 @@ subtype_dealloc.api_func.get_wrapper(space)) if space.is_w(w_type, space.w_str): pto.c_tp_itemsize = 1 + elif space.is_w(w_type, space.w_tuple): + pto.c_tp_itemsize = rffi.sizeof(PyObject) # buffer protocol setup_buffer_procs(space, w_type, pto) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -90,8 +90,9 @@ Py_DecRef(space, py_unicode.c_defenc) if py_unicode.c_str: lltype.free(py_unicode.c_str, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): From pypy.commits at gmail.com Mon Jun 27 11:51:23 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 27 Jun 2016 08:51:23 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <57714b7b.05261c0a.4a2f6.0f78@mx.google.com> Author: Matti Picus Branch: Changeset: r85404:f3a17a909ced Date: 2016-06-27 18:43 +0300 http://bitbucket.org/pypy/pypy/changeset/f3a17a909ced/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,8 @@ Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show the errno of the failing system call, but instead some random previous errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type From pypy.commits at gmail.com Mon Jun 27 11:52:32 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 08:52:32 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: old-style finalizers, recording Message-ID: <57714bc0.c8a61c0a.d017.0f34@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85405:e8bcbbe2187a Date: 2016-06-27 17:53 +0200 http://bitbucket.org/pypy/pypy/changeset/e8bcbbe2187a/ Log: old-style finalizers, recording diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -10,7 +10,7 @@ class BoehmGCTransformer(GCTransformer): malloc_zero_filled = True - FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void)) + FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Void)) def __init__(self, translator, inline=False): super(BoehmGCTransformer, self).__init__(translator, inline=inline) @@ -112,18 +112,18 @@ destrptr = None DESTR_ARG = None - if self.translator.config.translation.reverse_debugger: - destrptr = None # XXX for now - if destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value typename = TYPE.__name__ - def ll_finalizer(addr): + revdb = self.translator.config.translation.reverse_debugger + def ll_finalizer(gcref): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) - v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) + if revdb: + llop.revdb_call_destructor(lltype.Void, gcref) + v = lltype.cast_opaque_ptr(DESTR_ARG, gcref) ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) - fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) + fptr = self.annotate_finalizer(ll_finalizer, [llmemory.GCREF], lltype.Void) else: fptr = lltype.nullptr(self.FINALIZER_PTR.TO) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -574,6 +574,7 @@ 'revdb_watch_restore_state': LLOp(), 'revdb_weakref_create': LLOp(), 'revdb_weakref_deref': LLOp(), + 'revdb_call_destructor': LLOp(), } # ***** Run test_lloperation after changes. ***** diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -597,6 +597,9 @@ return self._op_boehm_malloc(op, 1) def OP_BOEHM_REGISTER_FINALIZER(self, op): + if self.db.reverse_debugger: + from rpython.translator.revdb import gencsupp + return gencsupp.boehm_register_finalizer(self, op) return 'GC_REGISTER_FINALIZER(%s, (GC_finalization_proc)%s, NULL, NULL, NULL);' \ % (self.expr(op.args[0]), self.expr(op.args[1])) diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -21,6 +21,10 @@ def record_malloc_uid(expr): return ' RPY_REVDB_REC_UID(%s);' % (expr,) +def boehm_register_finalizer(funcgen, op): + return 'rpy_reverse_db_register_destructor(%s, %s);' % ( + funcgen.expr(op.args[0]), funcgen.expr(op.args[1])) + def prepare_database(db): FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR, lltype.Ptr(rstr.STR)], diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -211,6 +211,8 @@ boehm_fq_trigger[i++](); } +static long in_invoke_finalizers; + static void record_stop_point(void) { /* Invoke the finalizers now. This will call boehm_fq_callback(), @@ -219,24 +221,54 @@ */ int i; char *p = rpy_rev_buffer; + int64_t done; rpy_reverse_db_flush(); - GC_invoke_finalizers(); fq_trigger(); - /* This should all be done without emitting anything to the rdb + /* This should be done without emitting anything to the rdb log. We check that, and emit just a ASYNC_FINALIZER_TRIGGER. */ if (current_packet_size() != 0) { fprintf(stderr, - "record_stop_point emitted unexpectedly to the rdb log\n"); + "record_stop_point emitted unexpected data into the rdb log\n"); exit(1); } *(int16_t *)p = ASYNC_FINALIZER_TRIGGER; memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t)); rpy_revdb.buf_p += sizeof(uint64_t); flush_buffer(); + + /* Invoke all Boehm finalizers. For new-style finalizers, this + will only cause them to move to the queues, where + boehm_fq_next_dead() will be able to fetch them later. For + old-style finalizers, this will really call the finalizers, + which first emit to the rdb log the uid of the object. So + after we do that any number of times, we emit the uid -1 to + mean "now done, continue with the rest of the program". + */ + in_invoke_finalizers++; + GC_invoke_finalizers(); + in_invoke_finalizers--; + RPY_REVDB_EMIT(done = -1;, int64_t _e, done); +} + +RPY_EXTERN +void rpy_reverse_db_call_destructor(void *obj) +{ + /* old-style finalizers. Should occur only from the + GC_invoke_finalizers() call above. + */ + int64_t uid; + + if (RPY_RDB_REPLAY) + return; + if (!in_invoke_finalizers) { + fprintf(stderr, "call_destructor: called at an unexpected time\n"); + exit(1); + } + RPY_REVDB_EMIT(uid = ((struct pypy_header0 *)obj)->h_uid;, int64_t _e, uid); } RPY_EXTERN @@ -462,7 +494,7 @@ static int last_recorded_breakpoint_num; static char breakpoint_mode; static uint64_t *future_ids, *future_next_id; -static void *finalizer_tree; +static void *finalizer_tree, *destructor_tree; static void attach_gdb(void) { @@ -649,7 +681,6 @@ exit(1); } - fetch_more_if_needed: keep = rpy_revdb.buf_readend - rpy_revdb.buf_p; assert(keep >= 0); @@ -681,7 +712,10 @@ } finalizer_trigger_saved_break = rpy_revdb.stop_point_break; rpy_revdb.stop_point_break = bp; - goto fetch_more_if_needed; + /* Now we should not fetch anything more until we reach + that finalizer_trigger_saved_break point. */ + rpy_revdb.buf_limit = rpy_revdb.buf_p; + return; default: fprintf(stderr, "bad packet header %d", (int)header); @@ -945,13 +979,12 @@ rpy_revdb.watch_enabled = any_watch_point; } +static void replay_call_destructors(void); + static void replay_stop_point(void) { - if (finalizer_trigger_saved_break != 0) { - rpy_revdb.stop_point_break = finalizer_trigger_saved_break; - finalizer_trigger_saved_break = 0; - fq_trigger(); - } + if (finalizer_trigger_saved_break != 0) + replay_call_destructors(); while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { save_state(); @@ -1140,4 +1173,85 @@ return result; } +struct destructor_s { + void *obj; + void (*callback)(void *); +}; + + static int _dtree_compare(const void *obj1, const void *obj2) +{ + const struct destructor_s *d1 = obj1; + const struct destructor_s *d2 = obj2; + const struct pypy_header0 *h1 = d1->obj; + const struct pypy_header0 *h2 = d2->obj; + if (h1->h_uid < h2->h_uid) + return -1; + if (h1->h_uid == h2->h_uid) + return 0; + else + return 1; +} + +RPY_EXTERN +void rpy_reverse_db_register_destructor(void *obj, void (*callback)(void *)) +{ + if (!RPY_RDB_REPLAY) { + GC_REGISTER_FINALIZER(obj, (GC_finalization_proc)callback, + NULL, NULL, NULL); + } + else { + struct destructor_s **item; + struct destructor_s *node = malloc(sizeof(struct destructor_s)); + if (!node) { + fprintf(stderr, "register_destructor: malloc: out of memory\n"); + exit(1); + } + node->obj = obj; + node->callback = callback; + item = tsearch(node, &destructor_tree, _dtree_compare); + if (item == NULL) { + fprintf(stderr, "register_destructor: tsearch: out of memory\n"); + exit(1); + } + if (*item != node) { + fprintf(stderr, "register_destructor: duplicate object\n"); + exit(1); + } + } +} + +static void replay_call_destructors(void) +{ + rpy_revdb.stop_point_break = finalizer_trigger_saved_break; + finalizer_trigger_saved_break = 0; + fq_trigger(); + + /* Re-enable fetching more, and fetch the uid's of objects + with old-style destructors that die now. + */ + rpy_reverse_db_fetch(__FILE__, __LINE__); + while (1) { + int64_t uid; + struct destructor_s d_dummy, *entry, **item; + struct pypy_header0 o_dummy; + RPY_REVDB_EMIT(abort();, int64_t _e, uid); + if (uid == -1) + break; + + d_dummy.obj = &o_dummy; + o_dummy.h_uid = uid; + item = tfind(&d_dummy, &destructor_tree, _dtree_compare); + if (item == NULL) { + fprintf(stderr, "next_dead: object not found\n"); + exit(1); + } + entry = *item; + assert(((struct pypy_header0 *)entry->obj)->h_uid == uid); + tdelete(entry, &destructor_tree, _dtree_compare); + + entry->callback(entry->obj); + free(entry); + } +} + /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -126,8 +126,9 @@ RPY_EXTERN void rpy_reverse_db_watch_restore_state(bool_t any_watch_point); RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target); RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref); -//RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); RPY_EXTERN int rpy_reverse_db_fq_register(void *obj); RPY_EXTERN void *rpy_reverse_db_next_dead(void *result); +RPY_EXTERN void rpy_reverse_db_register_destructor(void *obj, void(*)(void *)); +RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj); /* ------------------------------------------------------------ */ diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -28,6 +28,7 @@ def get_finalizer_queue_main(): from rpython.rtyper.lltypesystem import lltype, rffi + # from rpython.translator.tool.cbuild import ExternalCompilationInfo eci = ExternalCompilationInfo( pre_include_bits=["#define foobar(x) x\n"]) @@ -71,6 +72,36 @@ return 9 return main +def get_old_style_finalizer_main(): + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.translator.tool.cbuild import ExternalCompilationInfo + # + eci = ExternalCompilationInfo( + pre_include_bits=["#define foobar(x) x\n"]) + foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, + compilation_info=eci, _nowrapper=True) + class Glob: + pass + glob = Glob() + class X: + def __del__(self): + assert foobar(-7) == -7 + glob.count += 1 + def main(argv): + glob.count = 0 + lst = [X() for i in range(3000)] + x = -1 + for i in range(3000): + lst[i] = None + if i % 300 == 150: + rgc.collect() + revdb.stop_point() + x = glob.count + assert foobar(x) == x + print x + return 9 + return main + class TestRecording(BaseRecordingTests): @@ -174,47 +205,36 @@ total = intmask(total * 3 + d[uid]) assert total == expected - def test_finalizer_recorded(self): - py.test.skip("in-progress") - from rpython.rtyper.lltypesystem import lltype, rffi - from rpython.translator.tool.cbuild import ExternalCompilationInfo - eci = ExternalCompilationInfo( - pre_include_bits=["#define foobar(x) x\n"]) - foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed, - compilation_info=eci) - class Glob: - pass - glob = Glob() - class X: - def __del__(self): - glob.count += 1 - def main(argv): - glob.count = 0 - lst = [X() for i in range(3000)] - x = -1 - for i in range(3000): - lst[i] = None - if i % 300 == 150: - rgc.collect() - revdb.stop_point() - x = glob.count - assert foobar(x) == x - print x - return 9 + def test_old_style_finalizer(self): + main = get_old_style_finalizer_main() self.compile(main, backendopt=False) out = self.run('Xx') assert 1500 < int(out) <= 3000 rdb = self.fetch_rdb([self.exename, 'Xx']) - counts = [rdb.next() for i in range(3000)] - assert counts[0] >= 0 - for i in range(len(counts)-1): - assert counts[i] <= counts[i + 1] - assert counts[-1] == int(out) + seen_uids = set() + for i in range(3000): + triggered = False + if rdb.is_special_packet(): + time, = rdb.special_packet(ASYNC_FINALIZER_TRIGGER, 'q') + assert time == i + 1 + triggered = True + x = intmask(rdb.next()) + while True: + assert x != -1 + assert x not in seen_uids + seen_uids.add(x) + y = intmask(rdb.next()) + assert y == -7 # from the __del__ + x = intmask(rdb.next()) + if x == -1: + break + x = rdb.next() + assert x == len(seen_uids) + assert len(seen_uids) == int(out) # write() call x = rdb.next(); assert x == len(out) x = rdb.next('i'); assert x == 0 # errno x = rdb.next('q'); assert x == 3000 # number of stop points - assert rdb.done() class TestReplayingWeakref(InteractiveTests): From pypy.commits at gmail.com Mon Jun 27 12:05:55 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 09:05:55 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Replaying old-style finalizers Message-ID: <57714ee3.0410c20a.94f34.004b@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85406:060881a88f57 Date: 2016-06-27 18:07 +0200 http://bitbucket.org/pypy/pypy/changeset/060881a88f57/ Log: Replaying old-style finalizers diff --git a/rpython/memory/gctransform/support.py b/rpython/memory/gctransform/support.py --- a/rpython/memory/gctransform/support.py +++ b/rpython/memory/gctransform/support.py @@ -77,18 +77,23 @@ from rpython.rlib.rposix import c_write return c_write(fd, string, len(string)) +def destructor_failed(typename, e): + try: + write(2, "a destructor of type ") + write(2, typename) + write(2, " raised an exception ") + write(2, str(e)) + write(2, " ignoring it\n") + except: + pass +destructor_failed._dont_inline_ = True + def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) except Exception as e: - try: - write(2, "a destructor of type ") - write(2, typename) - write(2, " raised an exception ") - write(2, str(e)) - write(2, " ignoring it\n") - except: - pass + destructor_failed(typename, e) +ll_call_destructor._revdb_do_all_calls_ = True def ll_report_finalizer_error(e): try: diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -399,8 +399,13 @@ line += '\nPYPY_INHIBIT_TAIL_CALL();' break elif self.db.reverse_debugger: - from rpython.translator.revdb import gencsupp - line = gencsupp.emit(line, self.lltypename(v_result), r) + if getattr(getattr(self.graph, 'func', None), + '_revdb_do_all_calls_', False): + pass # a hack for ll_call_destructor() to mean + # that the calls should really be done + else: + from rpython.translator.revdb import gencsupp + line = gencsupp.emit(line, self.lltypename(v_result), r) return line def OP_DIRECT_CALL(self, op): diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1226,14 +1226,15 @@ finalizer_trigger_saved_break = 0; fq_trigger(); - /* Re-enable fetching more, and fetch the uid's of objects - with old-style destructors that die now. + /* Re-enable fetching (disabled when we saw ASYNC_FINALIZER_TRIGGER), + and fetch the uid's of dying objects with old-style destructors. */ rpy_reverse_db_fetch(__FILE__, __LINE__); while (1) { int64_t uid; struct destructor_s d_dummy, *entry, **item; struct pypy_header0 o_dummy; + RPY_REVDB_EMIT(abort();, int64_t _e, uid); if (uid == -1) break; @@ -1242,7 +1243,7 @@ o_dummy.h_uid = uid; item = tfind(&d_dummy, &destructor_tree, _dtree_compare); if (item == NULL) { - fprintf(stderr, "next_dead: object not found\n"); + fprintf(stderr, "replay_call_destructors: object not found\n"); exit(1); } entry = *item; diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -308,3 +308,18 @@ child = self.replay() child.send(Message(CMD_FORWARD, 3001)) child.expect(ANSWER_AT_END) + + +class TestReplayingOldStyleFinalizer(InteractiveTests): + expected_stop_points = 3000 + + def setup_class(cls): + from rpython.translator.revdb.test.test_basic import compile, run + main = get_old_style_finalizer_main() + compile(cls, main, backendopt=False) + run(cls, '') + + def test_replaying_old_style_finalizer(self): + child = self.replay() + child.send(Message(CMD_FORWARD, 3001)) + child.expect(ANSWER_AT_END) From pypy.commits at gmail.com Mon Jun 27 12:17:22 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 09:17:22 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Test fixes Message-ID: <57715192.cdcf1c0a.96a3e.fffff74e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85407:a6b2f857e5c0 Date: 2016-06-27 18:18 +0200 http://bitbucket.org/pypy/pypy/changeset/a6b2f857e5c0/ Log: Test fixes diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -186,14 +186,6 @@ } } -/* -RPY_EXTERN -void rpy_reverse_db_register_finalizer(void *obj, void (*finalizer)(void *)) -{ - ...; -} -*/ - void boehm_gc_finalizer_notifier(void) { /* This is called by Boehm when there are pending finalizers. diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -152,6 +152,9 @@ assert rdb.done() def test_finalizer_light_ignored(self): + py.test.skip("lightweight finalizers could be skipped, but that " + "requires also skipping (instead of recording) any " + "external call they do") class X: @rgc.must_be_light_finalizer def __del__(self): @@ -182,6 +185,7 @@ if rdb.is_special_packet(): time, = rdb.special_packet(ASYNC_FINALIZER_TRIGGER, 'q') assert time == i + 1 + y = intmask(rdb.next('q')); assert y == -1 triggered = True j = rdb.next() assert j == i + 1000000 * triggered From pypy.commits at gmail.com Mon Jun 27 12:33:57 2016 From: pypy.commits at gmail.com (arigo) Date: Mon, 27 Jun 2016 09:33:57 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Tweaks Message-ID: <57715575.48371c0a.9258c.2952@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85408:71052a487be1 Date: 2016-06-27 18:35 +0200 http://bitbucket.org/pypy/pypy/changeset/71052a487be1/ Log: Tweaks diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -21,7 +21,7 @@ #include "src-revdb/revdb_include.h" #define RDB_SIGNATURE "RevDB:" -#define RDB_VERSION 0x00FF0001 +#define RDB_VERSION 0x00FF0002 #define WEAKREF_AFTERWARDS_DEAD ((char)0xf2) #define WEAKREF_AFTERWARDS_ALIVE ((char)0xeb) @@ -58,7 +58,7 @@ RPY_RDB_REPLAY = replay_asked; #else if (replay_asked != RPY_RDB_REPLAY) { - fprintf(stderr, "This executable was only compiled for %s mode.", + fprintf(stderr, "This executable was only compiled for %s mode.\n", RPY_RDB_REPLAY ? "replay" : "record"); exit(1); } @@ -154,7 +154,14 @@ h.argc = argc; h.argv = argv; write_all((const char *)&h, sizeof(h)); + fprintf(stderr, "PID %d: recording revdb log to '%s'\n", + (int)getpid(), filename); } + else { + fprintf(stderr, "PID %d starting, log file disabled " + "(use PYPYRDB=logfile)\n", (int)getpid()); + } + rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t); rpy_revdb.buf_limit = rpy_rev_buffer + sizeof(rpy_rev_buffer) - 32; rpy_revdb.unique_id_seen = 1; @@ -710,7 +717,7 @@ return; default: - fprintf(stderr, "bad packet header %d", (int)header); + fprintf(stderr, "bad packet header %d\n", (int)header); exit(1); } } diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -22,7 +22,7 @@ # self.cur = 0 x = self.read1('c'); assert x == '\x00' - x = self.read1('P'); assert x == 0x00FF0001 + x = self.read1('P'); assert x == 0x00FF0002 x = self.read1('P'); assert x == 0 x = self.read1('P'); assert x == 0 self.argc = self.read1('P') From pypy.commits at gmail.com Mon Jun 27 12:34:08 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 27 Jun 2016 09:34:08 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Set oparg to valid value in case of dict unpack (max -> min), skip walkabout if keyword.arg is None in AST Message-ID: <57715580.4aa71c0a.6daac.2d30@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85409:7767a5f257ee Date: 2016-06-27 18:33 +0200 http://bitbucket.org/pypy/pypy/changeset/7767a5f257ee/ Log: Set oparg to valid value in case of dict unpack (max -> min), skip walkabout if keyword.arg is None in AST diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -3569,7 +3569,8 @@ def visit_sequence(self, seq): if seq is not None: for node in seq: - node.walkabout(self) + if node is not None: + node.walkabout(self) def visit_kwonlydefaults(self, seq): if seq is not None: diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1115,7 +1115,7 @@ # a new dict. If there is one dict and it's an unpacking, then #it needs to be copied into a new dict. while containers > 1 or is_unpacking: - oparg = max(containers, 255) + oparg = min(containers, 255) self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg) containers -= (oparg - 1) is_unpacking = 0 diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -279,7 +279,8 @@ self.emit("def visit_sequence(self, seq):", 1) self.emit("if seq is not None:", 2) self.emit("for node in seq:", 3) - self.emit("node.walkabout(self)", 4) + self.emit("if node is not None:", 4) + self.emit("node.walkabout(self)", 5) self.emit("") self.emit("def visit_kwonlydefaults(self, seq):", 1) self.emit("if seq is not None:", 2) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1369,16 +1369,8 @@ self.pushvalue(w_sum) def BUILD_MAP_UNPACK(self, itemcount, next_instr): - self.BUILD_MAP(itemcount, next_instr) - #w_sum = self.space.newdict() - #for i in range(itemcount, 0, -1): - # w_item = self.popvalue() - # #self.space.peek(i) - # self.space.call_method(w_sum, 'update', w_item) - ##while itemcount != 0: - ## self.popvalue() - ## itemcount -= 1 - #self.pushvalue(w_sum) + w_sum = self.unpack_helper(itemcount, next_instr) + self.pushvalue(self.space.newdict(w_sum)) ### ____________________________________________________________ ### class ExitFrame(Exception): From pypy.commits at gmail.com Mon Jun 27 13:22:45 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 27 Jun 2016 10:22:45 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement map_unpack opcode Message-ID: <577160e5.85c11c0a.1287f.66c5@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85410:5997558702eb Date: 2016-06-27 19:22 +0200 http://bitbucket.org/pypy/pypy/changeset/5997558702eb/ Log: Implement map_unpack opcode diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1369,8 +1369,19 @@ self.pushvalue(w_sum) def BUILD_MAP_UNPACK(self, itemcount, next_instr): - w_sum = self.unpack_helper(itemcount, next_instr) - self.pushvalue(self.space.newdict(w_sum)) + w_dict = self.space.newdict() + for i in range(itemcount, 0, -1): + w_item = self.peekvalue(i-1) + num_items = w_item.length() + for j in range(num_items): + (w_key, w_value) = w_item.popitem() + self.space.setitem(w_dict, w_key, w_value) + while itemcount != 0: + self.popvalue() + itemcount -= 1 + self.pushvalue(w_dict) + + ### ____________________________________________________________ ### class ExitFrame(Exception): From pypy.commits at gmail.com Mon Jun 27 13:37:49 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 27 Jun 2016 10:37:49 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <5771646d.c7a81c0a.1be5a.63ea@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85411:111ead0c2445 Date: 2016-06-27 18:37 +0100 http://bitbucket.org/pypy/pypy/changeset/111ead0c2445/ Log: hg merge default diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,14 @@ Allow rw access to the char* returned from PyString_AS_STRING, also refactor PyStringObject to look like cpython's and allow subclassing PyString_Type and PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -26,54 +26,10 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -120,8 +120,8 @@ def bytes_dealloc(space, py_obj): """Frees allocated PyBytesObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -58,6 +58,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -44,8 +44,8 @@ Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -4,6 +4,15 @@ #if PY_MAJOR_VERSION >= 3 #define PyInt_FromLong PyLong_FromLong #define PyInt_AsLong PyLong_AsLong + #define PyThing_FromStringAndSize PyUnicode_FromStringAndSize + #define PyThing_FromString PyUnicode_FromString + #define PyThing_Check PyUnicode_Check + #define _PyThing_AsString _PyUnicode_AsString +#else + #define PyThing_FromStringAndSize PyString_FromStringAndSize + #define PyThing_FromString PyString_FromString + #define PyThing_Check PyString_Check + #define _PyThing_AsString PyString_AsString #endif typedef struct { @@ -93,7 +102,7 @@ static PyObject * foo_get_name(PyObject *self, void *closure) { - return PyUnicode_FromStringAndSize("Foo Example", 11); + return PyThing_FromStringAndSize("Foo Example", 11); } static PyObject * @@ -119,7 +128,7 @@ { PyObject *format; - format = PyUnicode_FromString(""); + format = PyThing_FromString(""); if (format == NULL) return NULL; return format; } @@ -135,11 +144,11 @@ foo_setattro(fooobject *self, PyObject *name, PyObject *value) { char *name_str; - if (!PyUnicode_Check(name)) { + if (!PyThing_Check(name)) { PyErr_SetObject(PyExc_AttributeError, name); return -1; } - name_str = _PyUnicode_AsString(name); + name_str = _PyThing_AsString(name); if (strcmp(name_str, "set_foo") == 0) { long v = PyInt_AsLong(value); @@ -625,21 +634,39 @@ (destructor)custom_dealloc, /*tp_dealloc*/ }; +static PyTypeObject TupleLike = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.TupleLike", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; + + static PyObject *size_of_instances(PyObject *self, PyObject *t) { return PyInt_FromLong(((PyTypeObject *)t)->tp_basicsize); } + +static PyObject * is_TupleLike(PyObject *self, PyObject * t) +{ + int tf = t->ob_type == &TupleLike; + if (t->ob_type->tp_itemsize == 0) + return PyInt_FromLong(-1); + return PyInt_FromLong(tf); +} + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, {"newCustom", (PyCFunction)newCustom, METH_NOARGS, NULL}, {"size_of_instances", (PyCFunction)size_of_instances, METH_O, NULL}, + {"is_TupleLike", (PyCFunction)is_TupleLike, METH_O, NULL}, {NULL, NULL} /* Sentinel */ }; - +#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "foo", @@ -651,6 +678,7 @@ NULL, NULL, }; +#define INITERROR return NULL /* Initialize this module. */ #ifdef __GNUC__ @@ -661,8 +689,30 @@ PyMODINIT_FUNC PyInit_foo(void) + +#else + +#define INITERROR return + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +initfoo(void) +#endif { - PyObject *m, *d; + PyObject *d; +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("foo", foo_functions); +#endif + if (module == NULL) + INITERROR; footype.tp_new = PyType_GenericNew; @@ -671,53 +721,59 @@ MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) - return NULL; + INITERROR; if (PyType_Ready(&UnicodeSubtype) < 0) - return NULL; + INITERROR; if (PyType_Ready(&UnicodeSubtype2) < 0) - return NULL; + INITERROR; if (PyType_Ready(&MetaType) < 0) - return NULL; + INITERROR; if (PyType_Ready(&InitErrType) < 0) - return NULL; + INITERROR; if (PyType_Ready(&SimplePropertyType) < 0) - return NULL; + INITERROR; SimplePropertyType.tp_new = PyType_GenericNew; InitErrType.tp_new = PyType_GenericNew; CustomType.ob_type = &MetaType; if (PyType_Ready(&CustomType) < 0) - return NULL; + INITERROR; UnicodeSubtype3.tp_flags = Py_TPFLAGS_DEFAULT; UnicodeSubtype3.tp_base = &UnicodeSubtype; UnicodeSubtype3.tp_bases = Py_BuildValue("(OO)", &UnicodeSubtype, &CustomType); if (PyType_Ready(&UnicodeSubtype3) < 0) - return NULL; + INITERROR; - m = PyModule_Create(&moduledef); - if (m == NULL) - return NULL; - d = PyModule_GetDict(m); + TupleLike.tp_base = &PyTuple_Type; + if (PyType_Ready(&TupleLike) < 0) + INITERROR; + + + d = PyModule_GetDict(module); if (d == NULL) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype3", (PyObject *) &UnicodeSubtype3) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "InitErrType", (PyObject *) &InitErrType) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "Property", (PyObject *) &SimplePropertyType) < 0) - return NULL; + INITERROR; if (PyDict_SetItemString(d, "Custom", (PyObject *) &CustomType) < 0) - return NULL; - return m; + INITERROR; + if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) + INITERROR; +#if PY_MAJOR_VERSION >=3 + return module; +#endif } diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -17,24 +17,10 @@ """ PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); int result = 0; - size_t expected_size; if(PyByteArray_Size(s) == 12) { result = 1; } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*3; - #elif defined Py_DEBUG - expected_size = 64; - #else - expected_size = 48; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,24 +25,8 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyBytes_FromString("Hello world"); - int result = 0; - size_t expected_size; + int result = PyBytes_Size(s); - if(PyBytes_Size(s) == 11) { - result = 1; - } - #ifdef PYPY_VERSION - expected_size = 48; - #elif defined Py_DEBUG - expected_size = 53; - #else - expected_size = 37; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -51,7 +51,7 @@ api._PyTuple_Resize(ar, 10) assert api.PyTuple_Size(ar[0]) == 10 for i in range(3, 10): - rffi.cast(PyTupleObject, py_tuple).c_ob_item[i] = make_ref( + rffi.cast(PyTupleObject, ar[0]).c_ob_item[i] = make_ref( space, space.wrap(42 + i)) w_tuple = from_ref(space, ar[0]) assert space.int_w(space.len(w_tuple)) == 10 @@ -151,3 +151,8 @@ """), ]) module.run() + + def test_tuple_subclass(self): + module = self.import_module(name='foo') + a = module.TupleLike([1, 2, 3]) + assert module.is_TupleLike(a) == 1 diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -2,10 +2,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.debug import fatalerror_notb from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, + build_type_checkers, PyVarObjectFields, cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - make_ref, from_ref, decref, + make_ref, from_ref, decref, incref, pyobj_has_w_obj, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -29,8 +29,8 @@ PyTupleObjectStruct = lltype.ForwardReference() PyTupleObject = lltype.Ptr(PyTupleObjectStruct) ObjectItems = rffi.CArray(PyObject) -PyTupleObjectFields = PyObjectFields + \ - (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +PyTupleObjectFields = PyVarObjectFields + \ + (("ob_item", ObjectItems),) cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) @bootstrap_function @@ -56,14 +56,12 @@ tuple_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_tuple.layout.typedef) - py_obj = typedescr.allocate(space, space.w_tuple) + py_obj = typedescr.allocate(space, space.w_tuple, length) py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_ob_item = lltype.malloc(ObjectItems, length, - flavor='raw', zero=True, - add_memory_pressure=True) - py_tup.c_ob_size = length - return py_tup + p = py_tup.c_ob_item + for i in range(py_tup.c_ob_size): + p[i] = lltype.nullptr(PyObject.TO) + return py_obj def tuple_attach(space, py_obj, w_obj): """ @@ -71,23 +69,24 @@ buffer must not be modified. """ items_w = space.fixedview(w_obj) - l = len(items_w) - p = lltype.malloc(ObjectItems, l, flavor='raw', - add_memory_pressure=True) + py_tup = rffi.cast(PyTupleObject, py_obj) + length = len(items_w) + if py_tup.c_ob_size < length: + raise oefmt(space.w_ValueError, + "tuple_attach called on object with ob_size %d but trying to store %d", + py_tup.c_ob_size, length) i = 0 try: - while i < l: - p[i] = make_ref(space, items_w[i]) + while i < length: + py_tup.c_ob_item[i] = make_ref(space, items_w[i]) i += 1 except: while i > 0: i -= 1 - decref(space, p[i]) - lltype.free(p, flavor='raw') + ob = py_tup.c_ob_item[i] + py_tup.c_ob_item[i] = lltype.nullptr(PyObject.TO) + decref(space, ob) raise - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_ob_size = l - py_tup.c_ob_item = p def tuple_realize(space, py_obj): """ @@ -108,7 +107,9 @@ "converting a PyTupleObject into a W_TupleObject, " "but found NULLs as items") items_w[i] = w_item - w_obj = space.newtuple(items_w) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_TupleObject, w_type) + w_obj.__init__(items_w) track_reference(space, py_obj, w_obj) return w_obj @@ -118,18 +119,16 @@ """ py_tup = rffi.cast(PyTupleObject, py_obj) p = py_tup.c_ob_item - if p: - for i in range(py_tup.c_ob_size): - decref(space, p[i]) - lltype.free(p, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + for i in range(py_tup.c_ob_size): + decref(space, p[i]) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @cpython_api([Py_ssize_t], PyObject, result_is_ll=True) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return new_empty_tuple(space, size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PyTuple_SetItem(space, ref, index, py_obj): @@ -185,25 +184,25 @@ ref = p_ref[0] if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - ref = rffi.cast(PyTupleObject, ref) - oldsize = ref.c_ob_size - oldp = ref.c_ob_item - newp = lltype.malloc(ObjectItems, newsize, zero=True, flavor='raw', - add_memory_pressure=True) + oldref = rffi.cast(PyTupleObject, ref) + oldsize = oldref.c_ob_size + p_ref[0] = new_empty_tuple(space, newsize) + newref = rffi.cast(PyTupleObject, p_ref[0]) try: if oldsize < newsize: to_cp = oldsize else: to_cp = newsize for i in range(to_cp): - newp[i] = oldp[i] + ob = oldref.c_ob_item[i] + incref(space, ob) + newref.c_ob_item[i] = ob except: - lltype.free(newp, flavor='raw') + decref(space, p_ref[0]) + p_ref[0] = lltype.nullptr(PyObject.TO) raise - ref.c_ob_item = newp - ref.c_ob_size = newsize - lltype.free(oldp, flavor='raw') - # in this version, p_ref[0] never needs to be updated + finally: + decref(space, ref) return 0 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -371,6 +371,8 @@ # (minimally, if tp_basicsize is zero we copy it from the base) if not pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize + if pto.c_tp_itemsize < base_pto.c_tp_itemsize: + pto.c_tp_itemsize = base_pto.c_tp_itemsize flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) @@ -503,7 +505,7 @@ @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): - from pypy.module.cpyext.object import PyObject_dealloc + from pypy.module.cpyext.object import _dealloc obj_pto = rffi.cast(PyTypeObjectPtr, obj) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) @@ -514,7 +516,7 @@ heaptype = rffi.cast(PyHeapTypeObject, obj) Py_DecRef(space, heaptype.c_ht_name) Py_DecRef(space, base_pyo) - PyObject_dealloc(space, obj) + _dealloc(space, obj) def type_alloc(space, w_metatype, itemsize=0): @@ -565,6 +567,8 @@ subtype_dealloc.api_func.get_wrapper(space)) if space.is_w(w_type, space.w_str): pto.c_tp_itemsize = 1 + elif space.is_w(w_type, space.w_tuple): + pto.c_tp_itemsize = rffi.sizeof(PyObject) # buffer protocol if space.is_w(w_type, space.w_str): setup_bytes_buffer_procs(space, pto) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -85,8 +85,8 @@ lltype.free(py_unicode.c_buffer, flavor="raw") if py_unicode.c_utf8buffer: lltype.free(py_unicode.c_utf8buffer, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): diff --git a/rpython/doc/getting-started.rst b/rpython/doc/getting-started.rst --- a/rpython/doc/getting-started.rst +++ b/rpython/doc/getting-started.rst @@ -142,8 +142,8 @@ Translating Full Programs ~~~~~~~~~~~~~~~~~~~~~~~~~ -To translate full RPython programs, there is the script ``translate.py`` in -:source:`rpython/translator/goal`. Examples for this are a slightly changed version of +To translate full RPython programs, there is the script ``bin/rpython`` in +:source:`rpython/bin/`. Examples for this are a slightly changed version of Pystone:: python bin/rpython translator/goal/targetrpystonedalone diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -1,6 +1,7 @@ from rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from rpython.memory.gctransform.support import (get_rtti, - _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor) + _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor, + ll_report_finalizer_error) from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.flowspace.model import Constant from rpython.rtyper.lltypesystem.lloperation import llop @@ -58,6 +59,9 @@ self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() + self.finalizer_triggers = [] + self.finalizer_queue_indexes = {} # {fq: index} + def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): # XXX same behavior for zero=True: in theory that's wrong if TYPE._is_atomic(): @@ -115,6 +119,39 @@ self.finalizer_funcptrs[TYPE] = fptr return fptr + def get_finalizer_queue_index(self, hop): + fq_tag = hop.spaceop.args[0].value + assert 'FinalizerQueue TAG' in fq_tag.expr + fq = fq_tag.default + try: + index = self.finalizer_queue_indexes[fq] + except KeyError: + index = len(self.finalizer_queue_indexes) + assert index == len(self.finalizer_triggers) + # + def ll_finalizer_trigger(): + try: + fq.finalizer_trigger() + except Exception as e: + ll_report_finalizer_error(e) + ll_trigger = self.annotate_finalizer(ll_finalizer_trigger, [], + lltype.Void) + self.finalizer_triggers.append(ll_trigger) + self.finalizer_queue_indexes[fq] = index + return index + + def gct_gc_fq_register(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + v_ptr = hop.spaceop.args[1] + hop.genop("boehm_fq_register", [c_index, v_ptr]) + + def gct_gc_fq_next_dead(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + hop.genop("boehm_fq_next_dead", [c_index], + resultvar = hop.spaceop.result) + def gct_weakref_create(self, hop): v_instance, = hop.spaceop.args v_addr = hop.genop("cast_ptr_to_adr", [v_instance], diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -166,7 +166,19 @@ }) if '__int128_t' in rffi.TYPES: - _ctypes_cache[rffi.__INT128_T] = ctypes.c_longlong # XXX: Not right at all. But for some reason, It started by while doing JIT compile after a merge with default. Can't extend ctypes, because thats a python standard, right? + class c_int128(ctypes.Array): # based on 2 ulongs + _type_ = ctypes.c_uint64 + _length_ = 2 + @property + def value(self): + if sys.byteorder == 'little': + res = self[0] | (self[1] << 64) + else: + res = self[1] | (self[0] << 64) + if res >= (1 << 127): + res -= 1 << 128 + return res + _ctypes_cache[rffi.__INT128_T] = c_int128 # for unicode strings, do not use ctypes.c_wchar because ctypes # automatically converts arrays into unicode strings. diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -825,3 +825,13 @@ assert charp2str(p2) == "helLD" free_charp(p1) free_charp(p2) + +def test_sign_when_casting_uint_to_larger_int(): + from rpython.rtyper.lltypesystem import rffi + from rpython.rlib.rarithmetic import r_uint32, r_uint64 + # + value = 0xAAAABBBB + assert cast(lltype.SignedLongLong, r_uint32(value)) == value + if hasattr(rffi, '__INT128_T'): + value = 0xAAAABBBBCCCCDDDD + assert cast(rffi.__INT128_T, r_uint64(value)) == value diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py --- a/rpython/translator/backendopt/finalizer.py +++ b/rpython/translator/backendopt/finalizer.py @@ -36,8 +36,8 @@ raise FinalizerError(msg) else: result = self.analyze_direct_call(graph) - if result is self.top_result(): - log.red('old-style non-light finalizer: %r' % (graph,)) + #if result is self.top_result(): + # log.red('old-style non-light finalizer: %r' % (graph,)) return result def analyze_simple_operation(self, op, graphinfo): diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -212,6 +212,24 @@ compile_extra=['-DPYPY_USING_BOEHM_GC'], )) + gct = self.db.gctransformer + gct.finalizer_triggers = tuple(gct.finalizer_triggers) # stop changing + sourcelines = [''] + for trig in gct.finalizer_triggers: + sourcelines.append('RPY_EXTERN void %s(void);' % ( + self.db.get(trig),)) + sourcelines.append('') + sourcelines.append('void (*boehm_fq_trigger[])(void) = {') + for trig in gct.finalizer_triggers: + sourcelines.append('\t%s,' % (self.db.get(trig),)) + sourcelines.append('\tNULL') + sourcelines.append('};') + sourcelines.append('struct boehm_fq_s *boehm_fq_queues[%d];' % ( + len(gct.finalizer_triggers),)) + sourcelines.append('') + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=['\n'.join(sourcelines)])) + return eci def gc_startup_code(self): diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -121,7 +121,7 @@ #define OP_LLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ r = (long long)(((unsigned long long)(x)) << (y)) #define OP_LLLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, 128); \ - r = (__int128)(((unsigned __int128)(x)) << (y)) + r = (__int128_t)(((__uint128_t)(x)) << (y)) #define OP_UINT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ r = (x) << (y) @@ -157,7 +157,7 @@ #define OP_CAST_UINT_TO_INT(x,r) r = (Signed)(x) #define OP_CAST_INT_TO_UINT(x,r) r = (Unsigned)(x) #define OP_CAST_INT_TO_LONGLONG(x,r) r = (long long)(x) -#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128)(x) +#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128_t)(x) #define OP_CAST_CHAR_TO_INT(x,r) r = (Signed)((unsigned char)(x)) #define OP_CAST_INT_TO_CHAR(x,r) r = (char)(x) #define OP_CAST_PTR_TO_INT(x,r) r = (Signed)(x) /* XXX */ diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -73,9 +73,17 @@ #ifdef PYPY_USING_BOEHM_GC +struct boehm_fq_s { + void *obj; + struct boehm_fq_s *next; +}; +RPY_EXTERN void (*boehm_fq_trigger[])(void); + int boehm_gc_finalizer_lock = 0; void boehm_gc_finalizer_notifier(void) { + int i; + boehm_gc_finalizer_lock++; while (GC_should_invoke_finalizers()) { if (boehm_gc_finalizer_lock > 1) { @@ -86,6 +94,11 @@ } GC_invoke_finalizers(); } + + i = 0; + while (boehm_fq_trigger[i]) + boehm_fq_trigger[i++](); + boehm_gc_finalizer_lock--; } @@ -100,6 +113,28 @@ GC_finalize_on_demand = 1; GC_set_warn_proc(mem_boehm_ignore); } + +void boehm_fq_callback(void *obj, void *rawfqueue) +{ + struct boehm_fq_s **fqueue = rawfqueue; + struct boehm_fq_s *node = GC_malloc(sizeof(void *) * 2); + if (!node) + return; /* ouch, too bad */ + node->obj = obj; + node->next = *fqueue; + *fqueue = node; +} + +void *boehm_fq_next_dead(struct boehm_fq_s **fqueue) +{ + struct boehm_fq_s *node = *fqueue; + if (node != NULL) { + *fqueue = node->next; + return node->obj; + } + else + return NULL; +} #endif /* BOEHM GC */ diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -104,13 +104,21 @@ RPY_EXTERN int boehm_gc_finalizer_lock; RPY_EXTERN void boehm_gc_startup_code(void); RPY_EXTERN void boehm_gc_finalizer_notifier(void); +struct boehm_fq_s; +RPY_EXTERN struct boehm_fq_s *boehm_fq_queues[]; +RPY_EXTERN void (*boehm_fq_trigger[])(void); +RPY_EXTERN void boehm_fq_callback(void *, void *); +RPY_EXTERN void *boehm_fq_next_dead(struct boehm_fq_s **); #define OP_GC__DISABLE_FINALIZERS(r) boehm_gc_finalizer_lock++ #define OP_GC__ENABLE_FINALIZERS(r) (boehm_gc_finalizer_lock--, \ boehm_gc_finalizer_notifier()) -#define OP_GC_FQ_REGISTER(tag, obj, r) /* ignored so far */ -#define OP_GC_FQ_NEXT_DEAD(tag, r) (r = NULL) +#define OP_BOEHM_FQ_REGISTER(tagindex, obj, r) \ + GC_REGISTER_FINALIZER(obj, boehm_fq_callback, \ + boehm_fq_queues + tagindex, NULL, NULL) +#define OP_BOEHM_FQ_NEXT_DEAD(tagindex, r) \ + r = boehm_fq_next_dead(boehm_fq_queues + tagindex) #endif /* PYPY_USING_BOEHM_GC */ diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -393,20 +393,36 @@ assert res[3] == compute_hash(d) assert res[4] == compute_hash(("Hi", None, (7.5, 2, d))) - def test_finalizer_queue_is_at_least_ignored(self): + def test_finalizer_queue(self): class A(object): - pass + def __init__(self, i): + self.i = i + class Glob: + triggered = 0 + glob = Glob() class FQ(rgc.FinalizerQueue): Class = A + triggered = 0 def finalizer_trigger(self): - debug.debug_print("hello!") # not called so far + glob.triggered += 1 fq = FQ() # def fn(): - fq.register_finalizer(A()) + for i in range(1000): + fq.register_finalizer(A(i)) rgc.collect() rgc.collect() - fq.next_dead() + if glob.triggered == 0: + print "not triggered!" + return 50 + seen = {} + for i in range(1000): + a = fq.next_dead() + assert a.i not in seen + seen[a.i] = True + if len(seen) < 500: + print "seen only %d!" % len(seen) + return 51 return 42 f = self.getcompiled(fn) From pypy.commits at gmail.com Mon Jun 27 13:47:42 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 27 Jun 2016 10:47:42 -0700 (PDT) Subject: [pypy-commit] pypy default: fix for py3k Message-ID: <577166be.c7a81c0a.1be5a.6702@mx.google.com> Author: Ronan Lamy Branch: Changeset: r85412:080ef29ca064 Date: 2016-06-27 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/080ef29ca064/ Log: fix for py3k diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -6,7 +6,7 @@ #define PyInt_AsLong PyLong_AsLong #define PyThing_FromStringAndSize PyUnicode_FromStringAndSize #define PyThing_FromString PyUnicode_FromString - # defin PyThing_Check PyUnicode_Check + #define PyThing_Check PyUnicode_Check #define _PyThing_AsString _PyUnicode_AsString #else #define PyThing_FromStringAndSize PyString_FromStringAndSize @@ -749,7 +749,7 @@ TupleLike.tp_base = &PyTuple_Type; if (PyType_Ready(&TupleLike) < 0) - return; + INITERROR; d = PyModule_GetDict(module); From pypy.commits at gmail.com Mon Jun 27 14:06:53 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 27 Jun 2016 11:06:53 -0700 (PDT) Subject: [pypy-commit] pypy py3k: fix, enable tests Message-ID: <57716b3d.e153c20a.2ab5b.6af0@mx.google.com> Author: Matti Picus Branch: py3k Changeset: r85413:f9fa8c5ea722 Date: 2016-06-27 21:05 +0300 http://bitbucket.org/pypy/pypy/changeset/f9fa8c5ea722/ Log: fix, enable tests diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -28,7 +28,7 @@ int result = PyBytes_Size(s); Py_DECREF(s); - return PyBool_FromLong(result); + return PyLong_FromLong(result); """), ("test_Size_exception", "METH_NOARGS", """ diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test/test_structseq.py --- a/pypy/module/cpyext/test/test_structseq.py +++ b/pypy/module/cpyext/test/test_structseq.py @@ -6,7 +6,6 @@ class AppTestStructSeq(AppTestCpythonExtensionBase): def test_StructSeq(self): - skip("XXX: https://bugs.pypy.org/issue1557") module = self.import_extension('foo', prologue=""" #include From pypy.commits at gmail.com Mon Jun 27 14:18:47 2016 From: pypy.commits at gmail.com (mattip) Date: Mon, 27 Jun 2016 11:18:47 -0700 (PDT) Subject: [pypy-commit] pypy default: a passing test for binop_impl resolution order Message-ID: <57716e07.46461c0a.35152.483d@mx.google.com> Author: Matti Picus Branch: Changeset: r85414:f0b680130986 Date: 2016-06-27 20:02 +0300 http://bitbucket.org/pypy/pypy/changeset/f0b680130986/ Log: a passing test for binop_impl resolution order diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1858,6 +1858,56 @@ } } +static PyObject* +array_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int ii, nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int ii, nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2117,7 +2167,7 @@ 0, /* tp_setattr */ 0, /* tp_compare */ (reprfunc)array_repr, /* tp_repr */ - 0, /* tp_as_number*/ + &array_as_number, /* tp_as_number*/ &array_as_sequence, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ @@ -2126,7 +2176,8 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -77,3 +77,10 @@ #assert s == "carray\n_reconstruct\np0\n(S'i'\np1\n(lp2\nI1\naI2\naI3\naI4\natp3\nRp4\n." rra = pickle.loads(s) # rra is arr backwards #assert arr.tolist() == rra.tolist() + + def test_binop_mul_impl(self): + # check that rmul is called + module = self.import_module(name='array') + arr = module.array('i', [2]) + res = [1, 2, 3] * arr + assert res == [1, 2, 3, 1, 2, 3] From pypy.commits at gmail.com Mon Jun 27 14:41:46 2016 From: pypy.commits at gmail.com (rlamy) Date: Mon, 27 Jun 2016 11:41:46 -0700 (PDT) Subject: [pypy-commit] pypy default: Reduce diff with py3k Message-ID: <5771736a.4aa71c0a.6daac.585b@mx.google.com> Author: Ronan Lamy Branch: Changeset: r85415:65fe40eb8c0a Date: 2016-06-27 19:41 +0100 http://bitbucket.org/pypy/pypy/changeset/65fe40eb8c0a/ Log: Reduce diff with py3k diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -28,10 +28,10 @@ """ return PyBool_FromLong(PyByteArray_Check(PyTuple_GetItem(args, 0))); """)], prologue='#include ') - assert module.get_hello1() == 'Hello world' - assert module.get_hello2() == 'Hello world\x00' + assert module.get_hello1() == b'Hello world' + assert module.get_hello2() == b'Hello world\x00' assert module.test_Size() - assert module.test_is_bytearray(bytearray("")) + assert module.test_is_bytearray(bytearray(b"")) assert not module.test_is_bytearray(()) def test_bytearray_buffer_init(self): @@ -63,7 +63,7 @@ ]) s = module.getbytearray() assert len(s) == 4 - assert s == 'ab\x00c' + assert s == b'ab\x00c' def test_bytearray_mutable(self): module = self.import_extension('foo', [ @@ -79,9 +79,9 @@ """), ]) s = module.mutable() - if s == '\x00' * 10: + if s == b'\x00' * 10: assert False, "no RW access to bytearray" - assert s[:6] == 'works\x00' + assert s[:6] == b'works\x00' def test_AsByteArray(self): module = self.import_extension('foo', [ @@ -97,18 +97,18 @@ """), ]) s = module.getbytearray() - assert s == 'test' + assert s == b'test' def test_manipulations(self): import sys module = self.import_extension('foo', [ - ("bytearray_from_string", "METH_VARARGS", + ("bytearray_from_bytes", "METH_VARARGS", ''' - return PyByteArray_FromStringAndSize(PyString_AsString( + return PyByteArray_FromStringAndSize(PyBytes_AsString( PyTuple_GetItem(args, 0)), 4); ''' ), - ("str_from_bytearray", "METH_VARARGS", + ("bytes_from_bytearray", "METH_VARARGS", ''' char * buf; int n; @@ -121,7 +121,7 @@ return NULL; } n = PyByteArray_Size(obj); - return PyString_FromStringAndSize(buf, n); + return PyBytes_FromStringAndSize(buf, n); ''' ), ("concat", "METH_VARARGS", @@ -129,7 +129,7 @@ PyObject * ret, *right, *left; PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyUnicode_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -141,16 +141,16 @@ ret = PyByteArray_Concat(ba1, ba2); return ret; """)]) - assert module.bytearray_from_string("huheduwe") == "huhe" - assert module.str_from_bytearray(bytearray('abc')) == 'abc' + assert module.bytearray_from_bytes(b"huheduwe") == b"huhe" + assert module.bytes_from_bytearray(bytearray(b'abc')) == b'abc' if '__pypy__' in sys.builtin_module_names: # CPython only makes an assert. - raises(ValueError, module.str_from_bytearray, 4.0) - ret = module.concat('abc', 'def') - assert ret == 'abcdef' + raises(ValueError, module.bytes_from_bytearray, 4.0) + ret = module.concat(b'abc', b'def') + assert ret == b'abcdef' assert not isinstance(ret, str) assert isinstance(ret, bytearray) - raises(TypeError, module.concat, 'abc', u'def') + raises(TypeError, module.concat, b'abc', u'def') def test_bytearray_resize(self): module = self.import_extension('foo', [ @@ -159,7 +159,7 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyUnicode_FromString("parse failed"); } ba = PyByteArray_FromObject(obj); @@ -168,7 +168,7 @@ oldsize = PyByteArray_Size(ba); if (oldsize == 0) { - return PyString_FromString("oldsize is 0"); + return PyUnicode_FromString("oldsize is 0"); } ret = PyByteArray_Resize(ba, newsize); if (ret != 0) @@ -179,10 +179,9 @@ return ba; ''' )]) - ret = module.bytearray_resize('abc', 6) + ret = module.bytearray_resize(b'abc', 6) assert len(ret) == 6,"%s, len=%d" % (ret, len(ret)) - assert ret == 'abc\x00\x00\x00' - ret = module.bytearray_resize('abcdefghi', 4) + assert ret == b'abc\x00\x00\x00' + ret = module.bytearray_resize(b'abcdefghi', 4) assert len(ret) == 4,"%s, len=%d" % (ret, len(ret)) - assert ret == 'abcd' - + assert ret == b'abcd' diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -10,22 +10,22 @@ import py import sys -class AppTestStringObject(AppTestCpythonExtensionBase): - def test_stringobject(self): +class AppTestBytesObject(AppTestCpythonExtensionBase): + def test_bytesobject(self): module = self.import_extension('foo', [ ("get_hello1", "METH_NOARGS", """ - return PyString_FromStringAndSize( + return PyBytes_FromStringAndSize( "Hello world", 11); """), ("get_hello2", "METH_NOARGS", """ - return PyString_FromString("Hello world"); + return PyBytes_FromString("Hello world"); """), ("test_Size", "METH_NOARGS", """ - PyObject* s = PyString_FromString("Hello world"); - int result = PyString_Size(s); + PyObject* s = PyBytes_FromString("Hello world"); + int result = PyBytes_Size(s); Py_DECREF(s); return PyLong_FromLong(result); @@ -33,38 +33,38 @@ ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - PyString_Size(f); + PyBytes_Size(f); Py_DECREF(f); return NULL; """), - ("test_is_string", "METH_VARARGS", + ("test_is_bytes", "METH_VARARGS", """ - return PyBool_FromLong(PyString_Check(PyTuple_GetItem(args, 0))); + return PyBool_FromLong(PyBytes_Check(PyTuple_GetItem(args, 0))); """)], prologue='#include ') - assert module.get_hello1() == 'Hello world' - assert module.get_hello2() == 'Hello world' + assert module.get_hello1() == b'Hello world' + assert module.get_hello2() == b'Hello world' assert module.test_Size() == 11 raises(TypeError, module.test_Size_exception) - assert module.test_is_string("") - assert not module.test_is_string(()) + assert module.test_is_bytes(b"") + assert not module.test_is_bytes(()) - def test_string_buffer_init(self): + def test_bytes_buffer_init(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ PyObject *s, *t; char* c; - s = PyString_FromStringAndSize(NULL, 4); + s = PyBytes_FromStringAndSize(NULL, 4); if (s == NULL) return NULL; - t = PyString_FromStringAndSize(NULL, 3); + t = PyBytes_FromStringAndSize(NULL, 3); if (t == NULL) return NULL; Py_DECREF(t); - c = PyString_AS_STRING(s); + c = PyBytes_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -72,62 +72,62 @@ return s; """), ]) - s = module.getstring() + s = module.getbytes() assert len(s) == 4 - assert s == 'ab\x00c' + assert s == b'ab\x00c' - def test_string_tp_alloc(self): + def test_bytes_tp_alloc(self): module = self.import_extension('foo', [ ("tpalloc", "METH_NOARGS", """ PyObject *base; PyTypeObject * type; PyStringObject *obj; - base = PyString_FromString("test"); - if (PyString_GET_SIZE(base) != 4) - return PyLong_FromLong(-PyString_GET_SIZE(base)); + base = PyBytes_FromString("test"); + if (PyBytes_GET_SIZE(base) != 4) + return PyLong_FromLong(-PyBytes_GET_SIZE(base)); type = base->ob_type; if (type->tp_itemsize != 1) return PyLong_FromLong(type->tp_itemsize); obj = (PyStringObject*)type->tp_alloc(type, 10); - if (PyString_GET_SIZE(obj) != 10) - return PyLong_FromLong(PyString_GET_SIZE(obj)); - /* cannot work, there is only RO access */ - memcpy(PyString_AS_STRING(obj), "works", 6); + if (PyBytes_GET_SIZE(obj) != 10) + return PyLong_FromLong(PyBytes_GET_SIZE(obj)); + /* cannot work, there is only RO access + memcpy(PyBytes_AS_STRING(obj), "works", 6); */ Py_INCREF(obj); return (PyObject*)obj; """), ('alloc_rw', "METH_NOARGS", ''' - PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); - memcpy(PyString_AS_STRING(obj), "works", 6); + PyObject *obj = _PyObject_NewVar(&PyBytes_Type, 10); + memcpy(PyBytes_AS_STRING(obj), "works", 6); return (PyObject*)obj; '''), ]) s = module.alloc_rw() - assert s == 'works' + '\x00' * 5 + assert s == b'works' + b'\x00' * 5 s = module.tpalloc() - assert s == 'works' + '\x00' * 5 + assert s == b'\x00' * 10 def test_AsString(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyString_FromStringAndSize("test", 4); - char* c = PyString_AsString(s1); - PyObject* s2 = PyString_FromStringAndSize(c, 4); + PyObject* s1 = PyBytes_FromStringAndSize("test", 4); + char* c = PyBytes_AsString(s1); + PyObject* s2 = PyBytes_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), ]) - s = module.getstring() - assert s == 'test' + s = module.getbytes() + assert s == b'test' def test_manipulations(self): module = self.import_extension('foo', [ - ("string_as_string", "METH_VARARGS", + ("bytes_as_string", "METH_VARARGS", ''' - return PyString_FromStringAndSize(PyString_AsString( + return PyBytes_FromStringAndSize(PyBytes_AsString( PyTuple_GetItem(args, 0)), 4); ''' ), @@ -137,22 +137,22 @@ PyObject * left = PyTuple_GetItem(args, 0); Py_INCREF(left); /* the reference will be stolen! */ v = &left; - PyString_Concat(v, PyTuple_GetItem(args, 1)); + PyBytes_Concat(v, PyTuple_GetItem(args, 1)); return *v; """)]) - assert module.string_as_string("huheduwe") == "huhe" - ret = module.concat('abc', 'def') - assert ret == 'abcdef' + assert module.bytes_as_string(b"huheduwe") == b"huhe" + ret = module.concat(b'abc', b'def') + assert ret == b'abcdef' ret = module.concat('abc', u'def') assert not isinstance(ret, str) assert isinstance(ret, unicode) assert ret == 'abcdef' - def test_py_string_as_string_None(self): + def test_py_bytes_as_string_None(self): module = self.import_extension('foo', [ ("string_None", "METH_VARARGS", ''' - if (PyString_AsString(Py_None)) { + if (PyBytes_AsString(Py_None)) { Py_RETURN_NONE; } return NULL; @@ -162,18 +162,18 @@ def test_AsStringAndSize(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyString_FromStringAndSize("te\\0st", 5); + PyObject* s1 = PyBytes_FromStringAndSize("te\\0st", 5); char *buf; Py_ssize_t len; - if (PyString_AsStringAndSize(s1, &buf, &len) < 0) + if (PyBytes_AsStringAndSize(s1, &buf, &len) < 0) return NULL; if (len != 5) { PyErr_SetString(PyExc_AssertionError, "Bad Length"); return NULL; } - if (PyString_AsStringAndSize(s1, &buf, NULL) >= 0) { + if (PyBytes_AsStringAndSize(s1, &buf, NULL) >= 0) { PyErr_SetString(PyExc_AssertionError, "Should Have failed"); return NULL; } @@ -183,7 +183,7 @@ return Py_None; """), ]) - module.getstring() + module.getbytes() def test_py_string_as_string_Unicode(self): module = self.import_extension('foo', [ @@ -420,8 +420,8 @@ assert type(a).__name__ == 'string_' assert a == 'abc' -class TestString(BaseApiTest): - def test_string_resize(self, space, api): +class TestBytes(BaseApiTest): + def test_bytes_resize(self, space, api): py_str = new_empty_str(space, 10) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') py_str.c_ob_sval[0] = 'a' From pypy.commits at gmail.com Mon Jun 27 15:54:45 2016 From: pypy.commits at gmail.com (raffael_t) Date: Mon, 27 Jun 2016 12:54:45 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Temporary implementation for map_unpack_with_call (checks missing) Message-ID: <57718485.4a79c20a.9ccb.365d@mx.google.com> Author: Raffael Tfirst Branch: py3.5 Changeset: r85416:bab192d4f0e9 Date: 2016-06-27 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/bab192d4f0e9/ Log: Temporary implementation for map_unpack_with_call (checks missing) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1353,20 +1353,11 @@ w_sum = self.unpack_helper(itemcount, next_instr) self.pushvalue(self.space.newlist(w_sum)) - #TODO - #get intersection, store as setentry def BUILD_MAP_UNPACK_WITH_CALL(self, itemcount, next_instr): - w_sum = self.space.newdict() num_maps = itemcount & 0xff - function_location = (itemcount >> 8) & 0xff - for i in range(num_maps, 0, -1): - arg = self.space.peek(i) - # intersection = _dictviews_and(w_sum, arg) #check after bugs are done - self.space.call_method(w_sum, 'update', self.space.peek(i)) - while itemcount != 0: - self.popvalue() - itemcount -= 1 - self.pushvalue(w_sum) + w_dict = self.space.newdict() + import pdb; pdb.set_trace() + self.BUILD_MAP_UNPACK(num_maps, next_instr) def BUILD_MAP_UNPACK(self, itemcount, next_instr): w_dict = self.space.newdict() From pypy.commits at gmail.com Mon Jun 27 17:02:47 2016 From: pypy.commits at gmail.com (wlav) Date: Mon, 27 Jun 2016 14:02:47 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: start of exchanging reflex by cling Message-ID: <57719477.4e4ec20a.23eb8.3735@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85417:ff08d43af948 Date: 2016-06-07 11:14 -0700 http://bitbucket.org/pypy/pypy/changeset/ff08d43af948/ Log: start of exchanging reflex by cling From pypy.commits at gmail.com Mon Jun 27 17:02:50 2016 From: pypy.commits at gmail.com (wlav) Date: Mon, 27 Jun 2016 14:02:50 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: From Aditi: first stab at new Cling backend, based off the C++ Cppyy.cxx Message-ID: <5771947a.8438c20a.f0ac2.ffff9a88@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85418:1e669815cd70 Date: 2016-06-27 13:59 -0700 http://bitbucket.org/pypy/pypy/changeset/1e669815cd70/ Log: From Aditi: first stab at new Cling backend, based off the C++ Cppyy.cxx diff too long, truncating to 2000 out of 3541 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -77,3 +77,5 @@ ^.hypothesis/ ^release/ ^rpython/_cache$ + +pypy/module/cppyy/.+/*\.pcm diff --git a/pypy/module/cppyy/bench/Makefile b/pypy/module/cppyy/bench/Makefile --- a/pypy/module/cppyy/bench/Makefile +++ b/pypy/module/cppyy/bench/Makefile @@ -26,4 +26,4 @@ bench02Dict_reflex.so: bench02.h bench02.cxx bench02.xml $(genreflex) bench02.h $(genreflexflags) --selection=bench02.xml -I$(ROOTSYS)/include - g++ -o $@ bench02.cxx bench02_rflx.cpp -I$(ROOTSYS)/include -shared -lReflex -lHistPainter `root-config --libs` $(cppflags) $(cppflags2) + g++ -o $@ bench02.cxx bench02_rflx.cpp -I$(ROOTSYS)/include -shared -std=c++14 -lHistPainter `root-config --libs` $(cppflags) $(cppflags2) diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py --- a/pypy/module/cppyy/capi/__init__.py +++ b/pypy/module/cppyy/capi/__init__.py @@ -9,8 +9,8 @@ # the selection of the desired backend (default is Reflex). # choose C-API access method: -from pypy.module.cppyy.capi.loadable_capi import * -#from pypy.module.cppyy.capi.builtin_capi import * +#from pypy.module.cppyy.capi.loadable_capi import * +from pypy.module.cppyy.capi.builtin_capi import * from pypy.module.cppyy.capi.capi_types import C_OBJECT,\ C_NULL_TYPE, C_NULL_OBJECT diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py --- a/pypy/module/cppyy/capi/builtin_capi.py +++ b/pypy/module/cppyy/capi/builtin_capi.py @@ -1,7 +1,8 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import jit -import reflex_capi as backend +import cling_capi as backend +#import reflex_capi as backend #import cint_capi as backend from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\ diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py --- a/pypy/module/cppyy/capi/cling_capi.py +++ b/pypy/module/cppyy/capi/cling_capi.py @@ -16,7 +16,8 @@ if os.environ.get("ROOTSYS"): if config_stat != 0: # presumably Reflex-only rootincpath = [os.path.join(os.environ["ROOTSYS"], "interpreter/cling/include"), - os.path.join(os.environ["ROOTSYS"], "interpreter/llvm/inst/include")] + os.path.join(os.environ["ROOTSYS"], "interpreter/llvm/inst/include"), + os.path.join(os.environ["ROOTSYS"], "include"),] rootlibpath = [os.path.join(os.environ["ROOTSYS"], "lib64"), os.path.join(os.environ["ROOTSYS"], "lib")] else: rootincpath = [incdir] @@ -39,13 +40,21 @@ std_string_name = 'std::basic_string' +# force loading (and exposure) of libCore symbols +with rffi.scoped_str2charp('libCore.so') as ll_libname: + _coredll = rdynload.dlopen(ll_libname, rdynload.RTLD_GLOBAL | rdynload.RTLD_NOW) + +# require local translator path to pickup common defs +from rpython.translator import cdir +translator_c_dir = py.path.local(cdir) + eci = ExternalCompilationInfo( separate_module_files=[srcpath.join("clingcwrapper.cxx")], - include_dirs=[incpath] + rootincpath, + include_dirs=[incpath, translator_c_dir] + rootincpath, includes=["clingcwrapper.h"], library_dirs=rootlibpath, libraries=["Cling"], - compile_extra=["-fno-strict-aliasing"], + compile_extra=["-fno-strict-aliasing", "-std=c++14"], use_cpp_linker=True, ) diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -735,7 +735,7 @@ type_info = ( (rffi.LONG, ("long", "long int")), - (rffi.LONGLONG, ("long long", "long long int")), + (rffi.LONGLONG, ("long long", "long long int", "Long64_t")), ) for c_type, names in type_info: @@ -743,6 +743,7 @@ _immutable_ = True def __init__(self, space, default): self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default)) + class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter): _immutable_ = True libffitype = jit_libffi.types.pointer @@ -761,7 +762,7 @@ (rffi.USHORT, ("unsigned short", "unsigned short int")), (rffi.UINT, ("unsigned", "unsigned int")), (rffi.ULONG, ("unsigned long", "unsigned long int")), - (rffi.ULONGLONG, ("unsigned long long", "unsigned long long int")), + (rffi.ULONGLONG, ("unsigned long long", "unsigned long long int", "ULong64_t")), ) for c_type, names in type_info: diff --git a/pypy/module/cppyy/include/cpp_cppyy.h b/pypy/module/cppyy/include/cpp_cppyy.h new file mode 100644 --- /dev/null +++ b/pypy/module/cppyy/include/cpp_cppyy.h @@ -0,0 +1,143 @@ +#ifndef PYROOT_CPPYY_H +#define PYROOT_CPPYY_H + +// Standard +#include +#include +#include + +//ROOT types + typedef long Long_t; + typedef unsigned long ULong_t; + typedef long long Long64_t; + typedef unsigned long long ULong64_t; + typedef float Float_t; + typedef double Double_t; + typedef long double LongDouble_t; + typedef bool Bool_t; + typedef char Char_t; + typedef unsigned char UChar_t; + typedef short Short_t; + typedef unsigned short UShort_t; + typedef int Int_t; + typedef unsigned int UInt_t; + +namespace Cppyy { + typedef ptrdiff_t TCppScope_t; + typedef TCppScope_t TCppType_t; + typedef void* TCppObject_t; + typedef ptrdiff_t TCppMethod_t; + + typedef Long_t TCppIndex_t; + typedef void* (*TCppMethPtrGetter_t)( TCppObject_t ); + +// name to opaque C++ scope representation ----------------------------------- + TCppIndex_t GetNumScopes( TCppScope_t parent ); + std::string GetScopeName( TCppScope_t parent, TCppIndex_t iscope ); + std::string ResolveName( const std::string& cppitem_name ); + TCppScope_t GetScope( const std::string& scope_name ); + TCppType_t GetTemplate( const std::string& template_name ); + TCppType_t GetActualClass( TCppType_t klass, TCppObject_t obj ); + size_t SizeOf( TCppType_t klass ); + + Bool_t IsBuiltin( const std::string& type_name ); + Bool_t IsComplete( const std::string& type_name ); + + extern TCppScope_t gGlobalScope; // for fast access + +// memory management --------------------------------------------------------- + TCppObject_t Allocate( TCppType_t type ); + void Deallocate( TCppType_t type, TCppObject_t instance ); + TCppObject_t Construct( TCppType_t type ); + void Destruct( TCppType_t type, TCppObject_t instance ); + +// method/function dispatching ----------------------------------------------- + void CallV( TCppMethod_t method, TCppObject_t self, void* args ); + UChar_t CallB( TCppMethod_t method, TCppObject_t self, void* args ); + Char_t CallC( TCppMethod_t method, TCppObject_t self, void* args ); + Short_t CallH( TCppMethod_t method, TCppObject_t self, void* args ); + Int_t CallI( TCppMethod_t method, TCppObject_t self, void* args ); + Long_t CallL( TCppMethod_t method, TCppObject_t self, void* args ); + Long64_t CallLL( TCppMethod_t method, TCppObject_t self, void* args ); + Float_t CallF( TCppMethod_t method, TCppObject_t self, void* args ); + Double_t CallD( TCppMethod_t method, TCppObject_t self, void* args ); + LongDouble_t CallLD( TCppMethod_t method, TCppObject_t self, void* args ); + void* CallR( TCppMethod_t method, TCppObject_t self, void* args ); + Char_t* CallS( TCppMethod_t method, TCppObject_t self, void* args ); + TCppObject_t CallConstructor( TCppMethod_t method, TCppType_t type, void* args ); + void CallDestructor( TCppType_t type, TCppObject_t self ); + TCppObject_t CallO( TCppMethod_t method, TCppObject_t self, void* args, TCppType_t result_type ); + + TCppMethPtrGetter_t GetMethPtrGetter( TCppScope_t scope, TCppIndex_t imeth ); + +// handling of function argument buffer -------------------------------------- + void* AllocateFunctionArgs( size_t nargs ); + void DeallocateFunctionArgs( void* args ); + size_t GetFunctionArgSizeof(); + size_t GetFunctionArgTypeoffset(); + +// scope reflection information ---------------------------------------------- + Bool_t IsNamespace( TCppScope_t scope ); + Bool_t IsAbstract( TCppType_t type ); + Bool_t IsEnum( const std::string& type_name ); + +// class reflection information ---------------------------------------------- + std::string GetFinalName( TCppType_t type ); + std::string GetScopedFinalName( TCppType_t type ); + Bool_t HasComplexHierarchy( TCppType_t type ); + TCppIndex_t GetNumBases( TCppType_t type ); + std::string GetBaseName( TCppType_t type, TCppIndex_t ibase ); + Bool_t IsSubtype( TCppType_t derived, TCppType_t base ); + void AddSmartPtrType( const std::string& ); + Bool_t IsSmartPtr( const std::string& ); + +// calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 + ptrdiff_t GetBaseOffset( + TCppType_t derived, TCppType_t base, TCppObject_t address, int direction, bool rerror = false ); + +// method/function reflection information ------------------------------------ + TCppIndex_t GetNumMethods( TCppScope_t scope ); + TCppIndex_t GetMethodIndexAt( TCppScope_t scope, TCppIndex_t imeth ); + std::vector< TCppMethod_t > GetMethodsFromName( TCppScope_t scope, const std::string& name ); + + TCppMethod_t GetMethod( TCppScope_t scope, TCppIndex_t imeth ); + + std::string GetMethodName( TCppMethod_t ); + std::string GetMethodResultType( TCppMethod_t ); + TCppIndex_t GetMethodNumArgs( TCppMethod_t ); + TCppIndex_t GetMethodReqArgs( TCppMethod_t ); + std::string GetMethodArgName( TCppMethod_t, int iarg ); + std::string GetMethodArgType( TCppMethod_t, int iarg ); + std::string GetMethodArgDefault( TCppMethod_t, int iarg ); + std::string GetMethodSignature( TCppScope_t scope, TCppIndex_t imeth ); + Bool_t IsConstMethod( TCppMethod_t ); + + Bool_t IsMethodTemplate( TCppMethod_t ); + TCppIndex_t GetMethodNumTemplateArgs( TCppScope_t scope, TCppIndex_t imeth ); + std::string GetMethodTemplateArgName( TCppScope_t scope, TCppIndex_t imeth, TCppIndex_t iarg ); + + TCppIndex_t GetGlobalOperator( + TCppType_t scope, TCppType_t lc, TCppScope_t rc, const std::string& op ); + +// method properties --------------------------------------------------------- + Bool_t IsConstructor( TCppMethod_t method ); + Bool_t IsPublicMethod( TCppMethod_t method ); + Bool_t IsStaticMethod( TCppMethod_t method ); + +// data member reflection information ---------------------------------------- + TCppIndex_t GetNumDatamembers( TCppScope_t scope ); + std::string GetDatamemberName( TCppScope_t scope, TCppIndex_t idata ); + std::string GetDatamemberType( TCppScope_t scope, TCppIndex_t idata ); + ptrdiff_t GetDatamemberOffset( TCppScope_t scope, TCppIndex_t idata ); + TCppIndex_t GetDatamemberIndex( TCppScope_t scope, const std::string& name ); + +// data member properties ---------------------------------------------------- + Bool_t IsPublicData( TCppScope_t scope, TCppIndex_t idata ); + Bool_t IsStaticData( TCppScope_t scope, TCppIndex_t idata ); + Bool_t IsConstData( TCppScope_t scope, TCppIndex_t idata ); + Bool_t IsEnumData( TCppScope_t scope, TCppIndex_t idata ); + Int_t GetDimensionSize( TCppScope_t scope, TCppIndex_t idata, int dimension ); + +} // namespace Cppyy + +#endif // ifndef PYROOT_CPPYY_H diff --git a/pypy/module/cppyy/include/cppyy.h b/pypy/module/cppyy/include/cppyy.h --- a/pypy/module/cppyy/include/cppyy.h +++ b/pypy/module/cppyy/include/cppyy.h @@ -1,6 +1,8 @@ #ifndef CPPYY_CPPYY #define CPPYY_CPPYY +#include "cpp_cppyy.h" + #ifdef __cplusplus struct CPPYY_G__DUMMY_FOR_CINT7 { #else diff --git a/pypy/module/cppyy/src/callcontext.h b/pypy/module/cppyy/src/callcontext.h new file mode 100644 --- /dev/null +++ b/pypy/module/cppyy/src/callcontext.h @@ -0,0 +1,101 @@ +#ifndef PYROOT_TCALLCONTEXT_H +#define PYROOT_TCALLCONTEXT_H + +// Standard +#include + +//Bindings +#include "cpp_cppyy.h" + +//ROOT +#include "Rtypes.h" + +namespace PyROOT { + +// general place holder for function parameters + struct TParameter { + union Value { + Bool_t fBool; + Short_t fShort; + UShort_t fUShort; + Int_t fInt; + UInt_t fUInt; + Long_t fLong; + ULong_t fULong; + Long64_t fLongLong; + ULong64_t fULongLong; + Float_t fFloat; + Double_t fDouble; + LongDouble_t fLongDouble; + void* fVoidp; + } fValue; + void* fRef; + char fTypeCode; + }; + +// extra call information + struct TCallContext { + TCallContext( std::vector< TParameter >::size_type sz = 0 ) : fArgs( sz ), fFlags( 0 ) {} + + enum ECallFlags { + kNone = 0, + kIsSorted = 1, // if method overload priority determined + kIsCreator = 2, // if method creates python-owned objects + kIsConstructor = 4, // if method is a C++ constructor + kUseHeuristics = 8, // if method applies heuristics memory policy + kUseStrict = 16, // if method applies strict memory policy + kManageSmartPtr = 32, // if executor should manage smart pointers + kReleaseGIL = 64, // if method should release the GIL + kFast = 128, // if method should NOT handle signals + kSafe = 256 // if method should return on signals + }; + + // memory handling + static ECallFlags sMemoryPolicy; + static Bool_t SetMemoryPolicy( ECallFlags e ); + + // signal safety + static ECallFlags sSignalPolicy; + static Bool_t SetSignalPolicy( ECallFlags e ); + + // payload + std::vector< TParameter > fArgs; + UInt_t fFlags; + }; + + inline Bool_t IsSorted( UInt_t flags ) { + return flags & TCallContext::kIsSorted; + } + + inline Bool_t IsCreator( UInt_t flags ) { + return flags & TCallContext::kIsCreator; + } + + inline Bool_t IsConstructor( UInt_t flags ) { + return flags & TCallContext::kIsConstructor; + } + + inline Bool_t ManagesSmartPtr( TCallContext* ctxt ) { + return ctxt->fFlags & TCallContext::kManageSmartPtr; + } + + inline Bool_t ReleasesGIL( UInt_t flags ) { + return flags & TCallContext::kReleaseGIL; + } + + inline Bool_t ReleasesGIL( TCallContext* ctxt ) { + return ctxt ? (ctxt->fFlags & TCallContext::kReleaseGIL) : kFALSE; + } + + inline Bool_t UseStrictOwnership( TCallContext* ctxt ) { + if ( ctxt && (ctxt->fFlags & TCallContext::kUseStrict) ) + return kTRUE; + if ( ctxt && (ctxt->fFlags & TCallContext::kUseHeuristics) ) + return kFALSE; + + return TCallContext::sMemoryPolicy == TCallContext::kUseStrict; + } + +} // namespace PyROOT + +#endif // !PYROOT_TCALLCONTEXT_H diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -1,1810 +1,1364 @@ -#include "cppyy.h" -#include "clingcwrapper.h" +// Bindings +#include "capi.h" +#include "cpp_cppyy.h" +#include "callcontext.h" -/************************************************************************* - * Copyright (C) 1995-2014, the ROOT team. * - * LICENSE: LGPLv2.1; see http://root.cern.ch/drupal/content/license * - * CONTRIBUTORS: see http://root.cern.ch/drupal/content/contributors * - *************************************************************************/ +// ROOT +#include "TBaseClass.h" +#include "TClass.h" +#include "TClassRef.h" +#include "TClassTable.h" +#include "TClassEdit.h" +#include "TCollection.h" +#include "TDataMember.h" +#include "TDataType.h" +#include "TError.h" +#include "TFunction.h" +#include "TGlobal.h" +#include "TInterpreter.h" +#include "TList.h" +#include "TMethod.h" +#include "TMethodArg.h" +#include "TROOT.h" -#include +// Standard +#include +#include +#include +#include -#include "clang/AST/ASTContext.h" -#include "clang/AST/Decl.h" -#include "clang/AST/DeclBase.h" -#include "clang/AST/DeclCXX.h" -#include "clang/AST/PrettyPrinter.h" -#include "clang/AST/Type.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Sema/Sema.h" +// temp +#include +typedef PyROOT::TParameter TParameter; +// --temp -#include "cling/Interpreter/DynamicLibraryManager.h" -#include "cling/Interpreter/Interpreter.h" -#include "cling/Interpreter/LookupHelper.h" -#include "cling/Interpreter/StoredValueRef.h" -#include "cling/MetaProcessor/MetaProcessor.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/Support/raw_ostream.h" +// small number that allows use of stack for argument passing +const int SMALL_ARGS_N = 8; -#include -#include -#include -#include -#include +// data for life time management --------------------------------------------- +typedef std::vector< TClassRef > ClassRefs_t; +static ClassRefs_t g_classrefs( 1 ); +static const ClassRefs_t::size_type GLOBAL_HANDLE = 1; -#include -#include -#include +typedef std::map< std::string, ClassRefs_t::size_type > Name2ClassRefIndex_t; +static Name2ClassRefIndex_t g_name2classrefidx; -using namespace clang; +typedef std::map< Cppyy::TCppMethod_t, CallFunc_t* > Method2CallFunc_t; +static Method2CallFunc_t g_method2callfunc; +typedef std::vector< TFunction > GlobalFuncs_t; +static GlobalFuncs_t g_globalfuncs; -/* cling initialization --------------------------------------------------- */ +typedef std::vector< TGlobal* > GlobalVars_t; +static GlobalVars_t g_globalvars; + +// data ---------------------------------------------------------------------- +Cppyy::TCppScope_t Cppyy::gGlobalScope = GLOBAL_HANDLE; + +// smart pointer types +static std::set< std::string > gSmartPtrTypes = + { "auto_ptr", "shared_ptr", "weak_ptr", "unique_ptr" }; + + +// global initialization ----------------------------------------------------- namespace { -cling::Interpreter* gCppyy_Cling; -cling::MetaProcessor* gCppyy_MetaProcessor; +class ApplicationStarter { +public: + ApplicationStarter() { + // setup dummy holders for global and std namespaces + assert( g_classrefs.size() == GLOBAL_HANDLE ); + g_name2classrefidx[ "" ] = GLOBAL_HANDLE; + g_classrefs.push_back(TClassRef("")); + // ROOT ignores std/::std, so point them to the global namespace + g_name2classrefidx[ "std" ] = GLOBAL_HANDLE; + g_name2classrefidx[ "::std" ] = GLOBAL_HANDLE; + // add a dummy global to refer to as null at index 0 + g_globalvars.push_back( nullptr ); + } -struct Cppyy_InitCling { // TODO: check whether ROOT/meta's TCling is linked in - Cppyy_InitCling() { - std::vector cling_args_storage; - cling_args_storage.push_back("cling4cppyy"); - - // TODO: get this from env - cling_args_storage.push_back("-I/home/wlavrijsen/rootdev/root/etc"); - - std::vector interp_args; - for (std::vector::const_iterator iarg = cling_args_storage.begin(); - iarg != cling_args_storage.end(); ++iarg) - interp_args.push_back(iarg->c_str()); - - // TODO: get this from env - const char* llvm_resource_dir = "/home/wlavrijsen/rootdev/root/etc/cling"; - gCppyy_Cling = new cling::Interpreter( - interp_args.size(), &(interp_args[0]), llvm_resource_dir); - - // fInterpreter->installLazyFunctionCreator(llvmLazyFunctionCreator); - - { - // R__LOCKGUARD(gInterpreterMutex); - gCppyy_Cling->AddIncludePath("/home/wlavrijsen/rootdev/root/etc/cling"); - gCppyy_Cling->AddIncludePath("."); - } - - // don't check whether modules' files exist. - gCppyy_Cling->getCI()->getPreprocessorOpts().DisablePCHValidation = true; - - // Use a stream that doesn't close its file descriptor. - static llvm::raw_fd_ostream fMPOuts (STDOUT_FILENO, /* ShouldClose */ false); - gCppyy_MetaProcessor = new cling::MetaProcessor(*gCppyy_Cling, fMPOuts); - - gCppyy_Cling->enableDynamicLookup(); - } -} _init; - -typedef std::map NamedHandles_t; -static NamedHandles_t s_named; - -struct SimpleScope { - std::vector m_methods; - std::vector m_data; -}; - -typedef std::map Scopes_t; -static Scopes_t s_scopes; - -typedef std::map Wrappers_t; -static Wrappers_t s_wrappers; + ~ApplicationStarter() { + for ( auto ifunc : g_method2callfunc ) + gInterpreter->CallFunc_Delete( ifunc.second ); + } +} _applicationStarter; } // unnamed namespace - -/* local helpers --------------------------------------------------------- */ -static inline void print_error(const std::string& where, const std::string& what) { - std::cerr << where << ": " << what << std::endl; +// local helpers ------------------------------------------------------------- +static inline +TClassRef& type_from_handle( Cppyy::TCppScope_t scope ) +{ + assert( (ClassRefs_t::size_type) scope < g_classrefs.size() ); + return g_classrefs[ (ClassRefs_t::size_type)scope ]; } -static inline char* cppstring_to_cstring(const std::string& name) { +// type_from_handle to go here +static inline +TFunction* type_get_method( Cppyy::TCppType_t klass, Cppyy::TCppIndex_t idx ) +{ + TClassRef& cr = type_from_handle( klass ); + if ( cr.GetClass() ) + return (TFunction*)cr->GetListOfMethods()->At( idx ); + assert( klass == (Cppyy::TCppType_t)GLOBAL_HANDLE ); + return (TFunction*)idx; +} + +static inline +Cppyy::TCppScope_t declaring_scope( Cppyy::TCppMethod_t method ) +{ + TMethod* m = dynamic_cast( (TFunction*)method ); + if ( m ) return Cppyy::GetScope( m->GetClass()->GetName() ); + return (Cppyy::TCppScope_t)GLOBAL_HANDLE; +} + +static inline +char* cppstring_to_cstring(const std::string& name) { char* name_char = (char*)malloc(name.size() + 1); strcpy(name_char, name.c_str()); return name_char; } -static inline SimpleScope* scope_from_handle(cppyy_type_t handle) { - return s_scopes[(cppyy_scope_t)handle]; + +// name to opaque C++ scope representation ----------------------------------- +Cppyy::TCppIndex_t Cppyy::GetNumScopes( TCppScope_t scope ) +{ + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) return 0; // not supported if not at global scope + assert( scope == (TCppScope_t)GLOBAL_HANDLE ); + return gClassTable->Classes(); } -static inline std::string qualtype_to_string(const QualType& qt, const ASTContext& atx) { - std::string result; - - PrintingPolicy policy(atx.getPrintingPolicy()); - policy.SuppressTagKeyword = true; // no class or struct keyword - policy.SuppressScope = true; // force scope from a clang::ElaboratedType - policy.AnonymousTagLocations = false; // no file name + line number for anonymous types - // The scope suppression is required for getting rid of the anonymous part of the name - // of a class defined in an anonymous namespace. - - qt.getAsStringInternal(result, policy); - return result; +std::string Cppyy::GetScopeName( TCppScope_t parent, TCppIndex_t iscope ) +{ +// Retrieve the scope name of the scope indexed with iscope in parent. + TClassRef& cr = type_from_handle( parent ); + if ( cr.GetClass() ) return 0; // not supported if not at global scope + assert( parent == (TCppScope_t)GLOBAL_HANDLE ); + std::string name = gClassTable->At( iscope ); + if ( name.find("::") == std::string::npos ) + return name; + return ""; } -static inline std::vector build_args(int nargs, void* args) { - std::vector arguments; - arguments.reserve(nargs); - for (int i = 0; i < nargs; ++i) { - char tc = ((CPPYY_G__value*)args)[i].type; - if (tc != 'a' && tc != 'o') - arguments.push_back(&((CPPYY_G__value*)args)[i]); - else - arguments.push_back((void*)(*(long*)&((CPPYY_G__value*)args)[i])); - } - return arguments; +std::string Cppyy::ResolveName( const std::string& cppitem_name ) +{ +// Fully resolve the given name to the final type name. + std::string tclean = TClassEdit::CleanType( cppitem_name.c_str() ); + + TDataType* dt = gROOT->GetType( tclean.c_str() ); + if ( dt ) return dt->GetFullTypeName(); + return TClassEdit::ResolveTypedef( tclean.c_str(), true ); } +Cppyy::TCppScope_t Cppyy::GetScope( const std::string& sname ) +{ + std::string scope_name; + if ( sname.find( "std::", 0, 5 ) == 0 ) + scope_name = sname.substr( 5, std::string::npos ); + else + scope_name = sname; + scope_name = ResolveName( scope_name ); + auto icr = g_name2classrefidx.find( scope_name ); + if ( icr != g_name2classrefidx.end() ) + return (TCppType_t)icr->second; + + // use TClass directly, to enable auto-loading + TClassRef cr( TClass::GetClass( scope_name.c_str(), kTRUE, kTRUE ) ); + if ( !cr.GetClass() ) + return (TCppScope_t)NULL; + + // no check for ClassInfo as forward declared classes are okay (fragile) + + ClassRefs_t::size_type sz = g_classrefs.size(); + g_name2classrefidx[ scope_name ] = sz; + g_classrefs.push_back( TClassRef( scope_name.c_str() ) ); + return (TCppScope_t)sz; +} + +Cppyy::TCppType_t Cppyy::GetTemplate( const std::string& /* template_name */ ) +{ + return (TCppType_t)0; +} + +Cppyy::TCppType_t Cppyy::GetActualClass( TCppType_t klass, TCppObject_t obj ) +{ + TClassRef& cr = type_from_handle( klass ); + TClass* clActual = cr->GetActualClass( (void*)obj ); + if ( clActual && clActual != cr.GetClass() ) { + // TODO: lookup through name should not be needed + return (TCppType_t)GetScope( clActual->GetName() ); + } + return klass; +} + +size_t Cppyy::SizeOf( TCppType_t klass ) +{ + TClassRef& cr = type_from_handle( klass ); + if ( cr.GetClass() ) return (size_t)cr->Size(); + return (size_t)0; +} + +Bool_t Cppyy::IsBuiltin( const std::string& type_name ) +{ + TDataType* dt = gROOT->GetType( TClassEdit::CleanType( type_name.c_str(), 1 ).c_str() ); + if ( dt ) return dt->GetType() != kOther_t; + return kFALSE; +} + +Bool_t Cppyy::IsComplete( const std::string& type_name ) +{ +// verify whether the dictionary of this class is fully available + Bool_t b = kFALSE; + + Int_t oldEIL = gErrorIgnoreLevel; + gErrorIgnoreLevel = 3000; + TClass* klass = TClass::GetClass( TClassEdit::ShortType( type_name.c_str(), 1 ).c_str() ); + if ( klass && klass->GetClassInfo() ) // works for normal case w/ dict + b = gInterpreter->ClassInfo_IsLoaded( klass->GetClassInfo() ); + else { // special case for forward declared classes + ClassInfo_t* ci = gInterpreter->ClassInfo_Factory( type_name.c_str() ); + if ( ci ) { + b = gInterpreter->ClassInfo_IsLoaded( ci ); + gInterpreter->ClassInfo_Delete( ci ); // we own the fresh class info + } + } + gErrorIgnoreLevel = oldEIL; + return b; +} + +// memory management --------------------------------------------------------- +Cppyy::TCppObject_t Cppyy::Allocate( TCppType_t type ) +{ + TClassRef& cr = type_from_handle( type ); + return (TCppObject_t)malloc( cr->Size() ); +} + +void Cppyy::Deallocate( TCppType_t /* type */, TCppObject_t instance ) +{ + free( instance ); +} + +Cppyy::TCppObject_t Cppyy::Construct( TCppType_t type ) +{ + TClassRef& cr = type_from_handle( type ); + return (TCppObject_t)cr->New(); +} + +void Cppyy::Destruct( TCppType_t type, TCppObject_t instance ) +{ + TClassRef& cr = type_from_handle( type ); + cr->Destructor( (void*)instance ); +} + + +// method/function dispatching ----------------------------------------------- +static inline ClassInfo_t* GetGlobalNamespaceInfo() +{ + static ClassInfo_t* gcl = gInterpreter->ClassInfo_Factory(); + return gcl; +} + +static CallFunc_t* GetCallFunc( Cppyy::TCppMethod_t method ) +{ + auto icf = g_method2callfunc.find( method ); + if ( icf != g_method2callfunc.end() ) + return icf->second; + + CallFunc_t* callf = nullptr; + TFunction* func = (TFunction*)method; + std::string callString = ""; + +// create, if not cached + Cppyy::TCppScope_t scope = declaring_scope( method ); + const TClassRef& klass = type_from_handle( scope ); + if ( klass.GetClass() || (func && scope == GLOBAL_HANDLE) ) { + ClassInfo_t* gcl = klass.GetClass() ? klass->GetClassInfo() : nullptr; + if ( ! gcl ) + gcl = GetGlobalNamespaceInfo(); + + TCollection* method_args = func->GetListOfMethodArgs(); + TIter iarg( method_args ); + + TMethodArg* method_arg = 0; + while ((method_arg = (TMethodArg*)iarg.Next())) { + std::string fullType = method_arg->GetTypeNormalizedName(); + if ( callString.empty() ) + callString = fullType; + else + callString += ", " + fullType; + } + + Long_t offset = 0; + callf = gInterpreter->CallFunc_Factory(); + + gInterpreter->CallFunc_SetFuncProto( + callf, + gcl, + func ? func->GetName() : klass->GetName(), + callString.c_str(), + func ? (func->Property() & kIsConstMethod) : kFALSE, + &offset, + ROOT::kExactMatch ); + +// CLING WORKAROUND -- The number of arguments is not always correct (e.g. when there +// are default parameters, causing the callString to be wrong and +// the exact match to fail); or the method may have been inline or +// be compiler generated. In all those cases the exact match fails, +// whereas the conversion match sometimes works. + if ( ! gInterpreter->CallFunc_IsValid( callf ) ) { + gInterpreter->CallFunc_SetFuncProto( + callf, + gcl, + func ? func->GetName() : klass->GetName(), + callString.c_str(), + func ? (func->Property() & kIsConstMethod) : kFALSE, + &offset ); // <- no kExactMatch as that will fail + } +// -- CLING WORKAROUND + + } + + if ( !( callf && gInterpreter->CallFunc_IsValid( callf ) ) ) { + // TODO: propagate this error to caller w/o use of Python C-API + /* + PyErr_Format( PyExc_RuntimeError, "could not resolve %s::%s(%s)", + const_cast(klass).GetClassName(), + func ? func->GetName() : const_cast(klass).GetClassName(), + callString.c_str() ); */ + std::cerr << "TODO: report unresolved function error to Python\n"; + if ( callf ) gInterpreter->CallFunc_Delete( callf ); + return nullptr; + } + + g_method2callfunc[ method ] = callf; + return callf; +} + +static inline void copy_args( void* args_, void** vargs ) { + std::vector& args = *(std::vector*)args_; + for ( std::vector::size_type i = 0; i < args.size(); ++i ) { + switch ( args[i].fTypeCode ) { + case 'l': /* long */ + vargs[i] = (void*)&args[i].fValue.fLong; + break; + case 'f': /* double */ + vargs[i] = (void*)&args[i].fValue.fFloat; + break; + case 'd': /* double */ + vargs[i] = (void*)&args[i].fValue.fDouble; + break; + case 'D': /* long double */ + vargs[i] = (void*)&args[i].fValue.fLongDouble; + break; + case 'k': /* long long */ + case 'K': /* unsigned long long */ + case 'U': /* unsigned long */ + case 'p': /* void* */ + vargs[i] = (void*)&args[i].fValue.fVoidp; + break; + case 'V': /* (void*)type& */ + vargs[i] = args[i].fValue.fVoidp; + break; + case 'r': /* const type& */ + vargs[i] = args[i].fRef; + break; + default: + std::cerr << "unknown type code: " << args[i].fTypeCode << std::endl; + break; + } + } +} + +Bool_t FastCall( + Cppyy::TCppMethod_t method, void* args_, void* self, void* result ) +{ + const std::vector& args = *(std::vector*)args_; + + CallFunc_t* callf = GetCallFunc( method ); + if ( ! callf ) + return kFALSE; + + TInterpreter::CallFuncIFacePtr_t faceptr = gCling->CallFunc_IFacePtr( callf ); + if ( faceptr.fKind == TInterpreter::CallFuncIFacePtr_t::kGeneric ) { + if ( args.size() <= SMALL_ARGS_N ) { + void* smallbuf[SMALL_ARGS_N]; + copy_args( args_, smallbuf ); + faceptr.fGeneric( self, args.size(), smallbuf, result ); + } else { + std::vector buf( args.size() ); + copy_args( args_, buf.data() ); + faceptr.fGeneric( self, args.size(), buf.data(), result ); + } + return kTRUE; + } + + if ( faceptr.fKind == TInterpreter::CallFuncIFacePtr_t::kCtor ) { + if ( args.size() <= SMALL_ARGS_N ) { + void* smallbuf[SMALL_ARGS_N]; + copy_args( args_, (void**)smallbuf ); + faceptr.fCtor( (void**)smallbuf, result, args.size() ); + } else { + std::vector buf( args.size() ); + copy_args( args_, buf.data() ); + faceptr.fCtor( buf.data(), result, args.size() ); + } + return kTRUE; + } + + if ( faceptr.fKind == TInterpreter::CallFuncIFacePtr_t::kDtor ) { + std::cerr << " DESTRUCTOR NOT IMPLEMENTED YET! " << std::endl; + return kFALSE; + } + + return kFALSE; +} + +template< typename T > +static inline T CallT( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, void* args ) +{ + T t{}; + if ( FastCall( method, args, (void*)self, &t ) ) + return t; + return (T)-1; +} + +#define CPPYY_IMP_CALL( typecode, rtype ) \ +rtype Cppyy::Call##typecode( TCppMethod_t method, TCppObject_t self, void* args )\ +{ \ + return CallT< rtype >( method, self, args ); \ +} + +void Cppyy::CallV( TCppMethod_t method, TCppObject_t self, void* args ) +{ + if ( ! FastCall( method, args, (void*)self, nullptr ) ) + return /* TODO ... report error */; +} + +CPPYY_IMP_CALL( B, UChar_t ) +CPPYY_IMP_CALL( C, Char_t ) +CPPYY_IMP_CALL( H, Short_t ) +CPPYY_IMP_CALL( I, Int_t ) +CPPYY_IMP_CALL( L, Long_t ) +CPPYY_IMP_CALL( LL, Long64_t ) +CPPYY_IMP_CALL( F, Float_t ) +CPPYY_IMP_CALL( D, Double_t ) +CPPYY_IMP_CALL( LD, LongDouble_t ) + +void* Cppyy::CallR( TCppMethod_t method, TCppObject_t self, void* args ) +{ + void* r = nullptr; + if ( FastCall( method, args, (void*)self, &r ) ) + return r; + return nullptr; +} + +Char_t* Cppyy::CallS( TCppMethod_t method, TCppObject_t self, void* args ) +{ + Char_t* s = nullptr; + if ( FastCall( method, args, (void*)self, &s ) ) + return s; + return nullptr; +} + +Cppyy::TCppObject_t Cppyy::CallConstructor( + TCppMethod_t method, TCppType_t /* klass */, void* args ) { + void* obj = nullptr; + if ( FastCall( method, args, nullptr, &obj ) ) + return (TCppObject_t)obj; + return (TCppObject_t)0; +} + +void Cppyy::CallDestructor( TCppType_t type, TCppObject_t self ) +{ + TClassRef& cr = type_from_handle( type ); + cr->Destructor( (void*)self, kTRUE ); +} + +Cppyy::TCppObject_t Cppyy::CallO( TCppMethod_t method, + TCppObject_t self, void* args, TCppType_t result_type ) +{ + TClassRef& cr = type_from_handle( result_type ); + void* obj = malloc( cr->Size() ); + if ( FastCall( method, args, self, obj ) ) + return (TCppObject_t)obj; + return (TCppObject_t)0; +} + +Cppyy::TCppMethPtrGetter_t Cppyy::GetMethPtrGetter( + TCppScope_t /* scope */, TCppIndex_t /* imeth */ ) +{ + return (TCppMethPtrGetter_t)0; +} + + +// handling of function argument buffer -------------------------------------- +void* Cppyy::AllocateFunctionArgs( size_t nargs ) +{ + return new TParameter[nargs]; +} + +void Cppyy::DeallocateFunctionArgs( void* args ) +{ + delete [] (TParameter*)args; +} + +size_t Cppyy::GetFunctionArgSizeof() +{ + return sizeof( TParameter ); +} + +size_t Cppyy::GetFunctionArgTypeoffset() +{ + return offsetof( TParameter, fTypeCode ); +} + + +// scope reflection information ---------------------------------------------- +Bool_t Cppyy::IsNamespace( TCppScope_t scope ) { +// Test if this scope represents a namespace. + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) + return cr->Property() & kIsNamespace; + return kFALSE; +} + +Bool_t Cppyy::IsAbstract( TCppType_t klass ) { +// Test if this type may not be instantiated. + TClassRef& cr = type_from_handle( klass ); + if ( cr.GetClass() ) + return cr->Property() & kIsAbstract; + return kFALSE; +} + +Bool_t Cppyy::IsEnum( const std::string& type_name ) { + return gInterpreter->ClassInfo_IsEnum( type_name.c_str() ); +} + + +// class reflection information ---------------------------------------------- +std::string Cppyy::GetFinalName( TCppType_t klass ) +{ + if ( klass == GLOBAL_HANDLE ) // due to CLING WORKAROUND in InitConverters_ + return ""; + // TODO: either this or GetScopedFinalName is wrong + TClassRef& cr = type_from_handle( klass ); + return cr->GetName(); +} + +std::string Cppyy::GetScopedFinalName( TCppType_t klass ) +{ + // TODO: either this or GetFinalName is wrong + TClassRef& cr = type_from_handle( klass ); + return cr->GetName(); +} + +Bool_t Cppyy::HasComplexHierarchy( TCppType_t /* handle */ ) +{ +// Always TRUE for now (pre-empts certain optimizations). + return kTRUE; +} + +Cppyy::TCppIndex_t Cppyy::GetNumBases( TCppType_t klass ) +{ +// Get the total number of base classes that this class has. + TClassRef& cr = type_from_handle( klass ); + if ( cr.GetClass() && cr->GetListOfBases() != 0 ) + return cr->GetListOfBases()->GetSize(); + return 0; +} + +std::string Cppyy::GetBaseName( TCppType_t klass, TCppIndex_t ibase ) +{ + TClassRef& cr = type_from_handle( klass ); + return ((TBaseClass*)cr->GetListOfBases()->At( ibase ))->GetName(); +} + +Bool_t Cppyy::IsSubtype( TCppType_t derived, TCppType_t base ) +{ + if ( derived == base ) + return kTRUE; + TClassRef& derived_type = type_from_handle( derived ); + TClassRef& base_type = type_from_handle( base ); + return derived_type->GetBaseClass( base_type ) != 0; +} + +void Cppyy::AddSmartPtrType( const std::string& type_name ) { + gSmartPtrTypes.insert( ResolveName( type_name ) ); +} + +Bool_t Cppyy::IsSmartPtr( const std::string& type_name ) { +// checks if typename denotes a smart pointer +// TODO: perhaps make this stricter? + const std::string& real_name = ResolveName( type_name ); + return gSmartPtrTypes.find( + real_name.substr( 0,real_name.find( "<" ) ) ) != gSmartPtrTypes.end(); +} + +// type offsets -------------------------------------------------------------- +ptrdiff_t Cppyy::GetBaseOffset( TCppType_t derived, TCppType_t base, + TCppObject_t address, int direction, bool rerror ) +{ +// calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 + if ( derived == base || !(base && derived) ) + return (ptrdiff_t)0; + + TClassRef& cd = type_from_handle( derived ); + TClassRef& cb = type_from_handle( base ); + + if ( !cd.GetClass() || !cb.GetClass() ) + return (ptrdiff_t)0; + + Long_t offset = -1; + if ( ! (cd->GetClassInfo() && cb->GetClassInfo()) ) { // gInterpreter requirement + // would like to warn, but can't quite determine error from intentional + // hiding by developers, so only cover the case where we really should have + // had a class info, but apparently don't: + if ( cd->IsLoaded() ) { + // warn to allow diagnostics + std::ostringstream msg; + msg << "failed offset calculation between " << cb->GetName() << " and " << cd->GetName(); + // TODO: propagate this warning to caller w/o use of Python C-API + // PyErr_Warn( PyExc_RuntimeWarning, const_cast( msg.str().c_str() ) ); + std::cerr << "Warning: " << msg << '\n'; + } + + // return -1 to signal caller NOT to apply offset + return rerror ? (ptrdiff_t)offset : 0; + } + + offset = gInterpreter->ClassInfo_GetBaseOffset( + cd->GetClassInfo(), cb->GetClassInfo(), (void*)address, direction > 0 ); + if ( offset == -1 ) // Cling error, treat silently + return rerror ? (ptrdiff_t)offset : 0; + + return (ptrdiff_t)(direction < 0 ? -offset : offset); +} + + +// method/function reflection information ------------------------------------ +Cppyy::TCppIndex_t Cppyy::GetNumMethods( TCppScope_t scope ) +{ + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() && cr->GetListOfMethods() ) { + Cppyy::TCppIndex_t nMethods = (TCppIndex_t)cr->GetListOfMethods()->GetSize(); + if ( nMethods == (TCppIndex_t)0 ) { + std::string clName = GetScopedFinalName( scope ); + if ( clName.find( '<' ) != std::string::npos ) { + // chicken-and-egg problem: TClass does not know about methods until instantiation: force it + if ( TClass::GetClass( ("std::" + clName).c_str() ) ) + clName = "std::" + clName; + std::ostringstream stmt; + stmt << "template class " << clName << ";"; + gInterpreter->Declare( stmt.str().c_str() ); + // now reload the methods + return (TCppIndex_t)cr->GetListOfMethods( kTRUE )->GetSize(); + } + } + return nMethods; + } else if ( scope == (TCppScope_t)GLOBAL_HANDLE ) { + // enforce lazines by denying the existence of methods + return (TCppIndex_t)0; + } + return (TCppIndex_t)0; +} + +Cppyy::TCppIndex_t Cppyy::GetMethodIndexAt( TCppScope_t scope, TCppIndex_t imeth) +{ + TClassRef& cr = type_from_handle (scope); + if (cr.GetClass()) + return (TCppIndex_t)imeth; + assert(handle == (TCppType_t)GLOBAL_HANDLE); + return (TCppIndex_t)&g_globalfuncs[imeth]; +} + +std::vector< Cppyy::TCppMethod_t > Cppyy::GetMethodsFromName( + TCppScope_t scope, const std::string& name ) +{ +// TODO: this method assumes that the call for this name is made only +// once, and thus there is no need to store the results of the search +// in g_globalfuncs ... probably true, but needs verification + std::vector< TCppMethod_t > methods; + if ( scope == GLOBAL_HANDLE ) { + TCollection* funcs = gROOT->GetListOfGlobalFunctions( kTRUE ); + g_globalfuncs.reserve(funcs->GetSize()); + + TIter ifunc(funcs); + + TFunction* func = 0; + while ( (func = (TFunction*)ifunc.Next()) ) { + // cover not only direct matches, but also template matches + std::string fn = func->GetName(); + if ( fn.rfind( name, 0 ) == 0 ) { + // either match exactly, or match the name as template + if ( (name.size() == fn.size()) || + (name.size() < fn.size() && fn[name.size()] == '<') ) { + methods.push_back( (TCppMethod_t)func ); + } + } + } + } else { + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + // todo: handle overloads + TMethod* m = cr->GetMethodAny( name.c_str() ); + if ( m ) methods.push_back( (TCppMethod_t)m ); + } + } + + return methods; +} + +Cppyy::TCppMethod_t Cppyy::GetMethod( TCppScope_t scope, TCppIndex_t imeth ) +{ + TFunction* f = type_get_method( scope, imeth ); + return (Cppyy::TCppMethod_t)f; +} + +std::string Cppyy::GetMethodName( TCppMethod_t method ) +{ + if ( method ) { + std::string name = ((TFunction*)method)->GetName(); + //if ( IsMethodTemplate( method ) ) + // return name.substr( 0, name.find('<') ); + return name; + } + return ""; +} + +std::string Cppyy::GetMethodResultType( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + if ( f->ExtraProperty() & kIsConstructor ) + return "constructor"; + return f->GetReturnTypeNormalizedName(); + } + return ""; +} + +Cppyy::TCppIndex_t Cppyy::GetMethodNumArgs( TCppMethod_t method ) +{ + if ( method ) + return ((TFunction*)method)->GetNargs(); + return 0; +} + +Cppyy::TCppIndex_t Cppyy::GetMethodReqArgs( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + return (TCppIndex_t)(f->GetNargs() - f->GetNargsOpt()); + } + return (TCppIndex_t)0; +} + +std::string Cppyy::GetMethodArgName( TCppMethod_t method, int iarg ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + TMethodArg* arg = (TMethodArg*)f->GetListOfMethodArgs()->At( iarg ); + return arg->GetName(); + } + return ""; +} + +std::string Cppyy::GetMethodArgType( TCppMethod_t method, int iarg ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + TMethodArg* arg = (TMethodArg*)f->GetListOfMethodArgs()->At( iarg ); + return arg->GetTypeNormalizedName(); + } + return ""; +} + +std::string Cppyy::GetMethodArgDefault( TCppMethod_t method, int iarg ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + TMethodArg* arg = (TMethodArg*)f->GetListOfMethodArgs()->At( iarg ); + const char* def = arg->GetDefault(); + if ( def ) + return def; + } + + return ""; +} + +std::string Cppyy::GetMethodSignature( TCppScope_t /* scope */, TCppIndex_t /* imeth */ ) +{ + return ""; +} + +Bool_t Cppyy::IsConstMethod( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + return f->Property() & kIsConstMethod; + } + return kFALSE; +} + + +Bool_t Cppyy::IsMethodTemplate( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + std::string name = f->GetName(); + return (name[name.size()-1] == '>') && (name.find('<') != std::string::npos); + } + return kFALSE; +} + +Cppyy::TCppIndex_t Cppyy::GetMethodNumTemplateArgs( + TCppScope_t /* scope */, TCppIndex_t /* imeth */ ) +{ + return (TCppIndex_t)0; +} + +std::string Cppyy::GetMethodTemplateArgName( + TCppScope_t /* scope */, TCppIndex_t /* imeth */, TCppIndex_t /* iarg */ ) +{ + return ""; +} + +Cppyy::TCppIndex_t Cppyy::GetGlobalOperator( + TCppScope_t /* scope */, TCppType_t /* lc */, TCppType_t /* rc */, const std::string& /* op */ ) +{ + return (TCppIndex_t)0; +} + +// method properties --------------------------------------------------------- +Bool_t Cppyy::IsConstructor( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + return f->ExtraProperty() & kIsConstructor; + } + return kFALSE; +} + +Bool_t Cppyy::IsPublicMethod( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + return f->Property() & kIsPublic; + } + return kFALSE; +} + +Bool_t Cppyy::IsStaticMethod( TCppMethod_t method ) +{ + if ( method ) { + TFunction* f = (TFunction*)method; + return f->Property() & kIsStatic; + } + return kFALSE; +} + +// data member reflection information ---------------------------------------- +Cppyy::TCppIndex_t Cppyy::GetNumDatamembers( TCppScope_t scope ) +{ + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() && cr->GetListOfDataMembers() ) + return cr->GetListOfDataMembers()->GetSize(); + else if ( scope == (TCppScope_t)GLOBAL_HANDLE ) { + std::cerr << " global data should be retrieved lazily " << std::endl; + TCollection* vars = gROOT->GetListOfGlobals( kTRUE ); + if ( g_globalvars.size() != (GlobalVars_t::size_type)vars->GetSize() ) { + g_globalvars.clear(); + g_globalvars.reserve(vars->GetSize()); + + TIter ivar(vars); + + TGlobal* var = 0; + while ( (var = (TGlobal*)ivar.Next()) ) + g_globalvars.push_back( var ); + } + return (TCppIndex_t)g_globalvars.size(); + } + return (TCppIndex_t)0; +} + +std::string Cppyy::GetDatamemberName( TCppScope_t scope, TCppIndex_t idata ) +{ + TClassRef& cr = type_from_handle( scope ); + if (cr.GetClass()) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->GetName(); + } + assert( scope == (TCppScope_t)GLOBAL_HANDLE ); + TGlobal* gbl = g_globalvars[ idata ]; + return gbl->GetName(); +} + +std::string Cppyy::GetDatamemberType( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gbl = g_globalvars[ idata ]; + std::string fullType = gbl->GetFullTypeName(); + if ( fullType[fullType.size()-1] == '*' && \ + fullType.find( "char", 0, 4 ) == std::string::npos ) + fullType.append( "*" ); + else if ( (int)gbl->GetArrayDim() > 1 ) + fullType.append( "*" ); + else if ( (int)gbl->GetArrayDim() == 1 ) { + std::ostringstream s; + s << '[' << gbl->GetMaxIndex( 0 ) << ']' << std::ends; + fullType.append( s.str() ); + } + return fullType; + } + + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + std::string fullType = m->GetTrueTypeName(); + if ( (int)m->GetArrayDim() > 1 || (!m->IsBasic() && m->IsaPointer()) ) + fullType.append( "*" ); + else if ( (int)m->GetArrayDim() == 1 ) { + std::ostringstream s; + s << '[' << m->GetMaxIndex( 0 ) << ']' << std::ends; + fullType.append( s.str() ); + } + return fullType; + } + + return ""; +} + +ptrdiff_t Cppyy::GetDatamemberOffset( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gbl = g_globalvars[ idata ]; + return (ptrdiff_t)gbl->GetAddress(); + } + + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return (ptrdiff_t)m->GetOffsetCint(); // yes, CINT ... + } + + return (ptrdiff_t)0; +} + +Cppyy::TCppIndex_t Cppyy::GetDatamemberIndex( TCppScope_t scope, const std::string& name ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gb = (TGlobal*)gROOT->GetListOfGlobals( kTRUE )->FindObject( name.c_str() ); + if ( gb && gb->GetAddress() && gb->GetAddress() != (void*)-1 ) { + g_globalvars.push_back( gb ); + return g_globalvars.size() - 1; + } + + } else { + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* dm = + (TDataMember*)cr->GetListOfDataMembers()->FindObject( name.c_str() ); + // TODO: turning this into an index is silly ... + if ( dm ) return (TCppIndex_t)cr->GetListOfDataMembers()->IndexOf( dm ); + } + } + + return (TCppIndex_t)-1; +} + + +// data member properties ---------------------------------------------------- +Bool_t Cppyy::IsPublicData( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) + return kTRUE; + TClassRef& cr = type_from_handle( scope ); + if ( cr->Property() & kIsNamespace ) + return kTRUE; + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->Property() & kIsPublic; +} + +Bool_t Cppyy::IsStaticData( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) + return kTRUE; + TClassRef& cr = type_from_handle( scope ); + if ( cr->Property() & kIsNamespace ) + return kTRUE; + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->Property() & kIsStatic; +} + +Bool_t Cppyy::IsConstData( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gbl = g_globalvars[ idata ]; + return gbl->Property() & kIsConstant; + } + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->Property() & kIsConstant; + } + return kFALSE; +} + +Bool_t Cppyy::IsEnumData( TCppScope_t scope, TCppIndex_t idata ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gbl = g_globalvars[ idata ]; + return gbl->Property() & kIsEnum; + } + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->Property() & kIsEnum; + } + return kFALSE; +} + +Int_t Cppyy::GetDimensionSize( TCppScope_t scope, TCppIndex_t idata, int dimension ) +{ + if ( scope == GLOBAL_HANDLE ) { + TGlobal* gbl = g_globalvars[ idata ]; + return gbl->GetMaxIndex( dimension ); + } + TClassRef& cr = type_from_handle( scope ); + if ( cr.GetClass() ) { + TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At( idata ); + return m->GetMaxIndex( dimension ); + } + return (Int_t)-1; +} + + +static inline +std::vector vsargs_to_parvec(void* args, int nargs) +{ + std::vector v; + v.reserve(nargs); + for (int i=0; igetLookupHelper(); - const Type* type = 0; - const Decl* decl = lh.findScope(scope_name, &type, /* intantiateTemplate= */ true); - if (!decl) { - //std::string buf = TClassEdit::InsertStd(name); - //decl = lh.findScope(buf, &type, /* intantiateTemplate= */ true); - } - if (!decl && type) { - const TagType* tagtype = type->getAs(); - if (tagtype) { - decl = tagtype->getDecl(); - } - } - - std::cout << "FOR: " << scope_name << " RECEIVED: " << type << " AND: " << decl << std::endl; - if (decl) { - DeclContext* dc = llvm::cast(const_cast(decl)); - SimpleScope* s = new SimpleScope; - for (DeclContext::decl_iterator idecl = dc->decls_begin(); *idecl; ++idecl) { - if (FunctionDecl* m = llvm::dyn_cast_or_null(*idecl)) - s->m_methods.push_back(m); - else if (FieldDecl* d = llvm::dyn_cast_or_null(*idecl)) - s->m_data.push_back(d); - } - s_scopes[(cppyy_scope_t)decl] = s; - } - - return (cppyy_scope_t)decl; // lookup failure return 0 (== error) + return cppyy_scope_t(Cppyy::GetScope(scope_name)); } +cppyy_type_t cppyy_get_template(const char* template_name) { + return cppyy_type_t(Cppyy::GetTemplate(template_name)); +} + +cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) { + return cppyy_type_t(Cppyy::GetActualClass(klass, (void*)obj)); +} + + +/* memory management ------------------------------------------------------ */ +cppyy_object_t cppyy_allocate(cppyy_type_t type) { + return cppyy_object_t(Cppyy::Allocate(type)); +} + +void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self) { + Cppyy::Deallocate(type, (void*)self); +} + +void cppyy_destruct(cppyy_type_t type, cppyy_object_t self) { + Cppyy::Destruct(type, (void*)self); +} + /* method/function dispatching -------------------------------------------- */ - -// TODO: expect the below to live in libCling.so -static CPPYY_Cling_Wrapper_t make_wrapper(const FunctionDecl* fdecl); -static void exec_with_valref_return(void* address, cling::StoredValueRef* ret, const FunctionDecl*); -static long long sv_to_long_long(const cling::StoredValueRef& svref); -// -- TODO: expect the above to live in libCling.so - - -template -static inline T cppyy_call_T(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { - if (s_wrappers.find(method) == s_wrappers.end()) { - make_wrapper((FunctionDecl*)method); - } - cling::StoredValueRef ret; - // std::vector arguments = build_args(nargs, args); - // CPPYY_Cling_Wrapper_t cb = (CPPYY_Cling_Wrapper_t)method; - exec_with_valref_return((void*)self, &ret, (FunctionDecl*)method); - // (*cb)((void*)self, nargs, const_cast(arguments.data()), ret); - return static_cast(sv_to_long_long(ret)); +void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + Cppyy::CallV(method, (void*)self, &parvec); } +unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (unsigned char)Cppyy::CallB(method, (void*)self, &parvec); +} +char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (char)Cppyy::CallC(method, (void*)self, &parvec); +} + +short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (short)Cppyy::CallH(method, (void*)self, &parvec); +} int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { - return cppyy_call_T(method, self, nargs, args); + std::vector parvec = vsargs_to_parvec(args, nargs); + return (int)Cppyy::CallI(method, (void*)self, &parvec); } - - -cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { - return (cppyy_methptrgetter_t)0; +long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args){ + std::vector parvec = vsargs_to_parvec(args, nargs); + return (long)Cppyy::CallL(method, (void*)self, &parvec); } +long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (long long)Cppyy::CallLL(method, (void*)self, &parvec); +} + +float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (float)Cppyy::CallF(method, (void*)self, &parvec); +} + +double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (double)Cppyy::CallD(method, (void*)self, &parvec); +} + +void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return (void*)Cppyy::CallR(method, (void*)self, &parvec); +} + +char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return cppstring_to_cstring(Cppyy::CallS(method, (void*)self, &parvec)); +} + +cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return cppyy_object_t(Cppyy::CallConstructor(method, klass, &parvec)); +// return cppyy_object_t(Cppyy::CallConstructor(method, klass, args)); +} + +cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type) { + std::vector parvec = vsargs_to_parvec(args, nargs); + return cppyy_object_t(Cppyy::CallO(method, (void*)self, &parvec, result_type)); +} + +cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx) { + return cppyy_methptrgetter_t(Cppyy::GetMethPtrGetter(scope, idx)); +} + /* handling of function argument buffer ----------------------------------- */ -void* cppyy_allocate_function_args(size_t nargs) { - CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value)); - for (size_t i = 0; i < nargs; ++i) - args[i].type = 'l'; - return (void*)args; +void* cppyy_allocate_function_args(int nargs){ + return (void*)Cppyy::AllocateFunctionArgs(nargs); } -void cppyy_deallocate_function_args(void* args) { - free(args); +void cppyy_deallocate_function_args(void* args){ + Cppyy::DeallocateFunctionArgs(args); } -size_t cppyy_function_arg_sizeof() { - return sizeof(CPPYY_G__value); +size_t cppyy_function_arg_sizeof(){ + return (size_t)Cppyy::GetFunctionArgSizeof(); } -size_t cppyy_function_arg_typeoffset() { - return offsetof(CPPYY_G__value, type); +size_t cppyy_function_arg_typeoffset(){ + return (size_t)Cppyy::GetFunctionArgTypeoffset(); } /* scope reflection information ------------------------------------------- */ -int cppyy_is_namespace(cppyy_scope_t /* handle */) { - return 0; -} - -int cppyy_is_enum(const char* /* type_name */) { - return 0; -} - - -/* class reflection information ------------------------------------------- */ -char* cppyy_final_name(cppyy_type_t handle) { - for (NamedHandles_t::iterator isp = s_named.begin(); isp != s_named.end(); ++isp) { - if (isp->second == (cppyy_scope_t)handle) - return cppstring_to_cstring(isp->first); - } - return cppstring_to_cstring(""); +int cppyy_is_namespace(cppyy_scope_t scope) { + return (int)Cppyy::IsNamespace(scope); } -char* cppyy_scoped_final_name(cppyy_type_t handle) { - return cppyy_final_name(handle); -} - -int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) { - return 1; +int cppyy_is_enum(const char* type_name){ + return (int)Cppyy::IsEnum(type_name); } -/* method/function reflection information --------------------------------- */ -int cppyy_num_methods(cppyy_scope_t handle) { - SimpleScope* s = scope_from_handle(handle); - if (!s) return 0; - return s->m_methods.size(); +/* class reflection information ------------------------------------------- */ +char* cppyy_final_name(cppyy_type_t type) { + return cppstring_to_cstring(Cppyy::GetFinalName(type)); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; +char* cppyy_scoped_final_name(cppyy_type_t type) { + return cppstring_to_cstring(Cppyy::GetScopedFinalName(type)); } -char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { - SimpleScope* s = scope_from_handle(handle); - if (!s) return cppstring_to_cstring(""); - FunctionDecl* meth = s->m_methods.at(method_index); - std::cout << " METHOD NAME: " << meth->getDeclName().getAsString() << std::endl; - return cppstring_to_cstring(meth->getDeclName().getAsString()); +int cppyy_has_complex_hierarchy(cppyy_type_t type) { + return (int)Cppyy::HasComplexHierarchy(type); } -char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) { - SimpleScope* s = scope_from_handle(handle); - if (!s) return cppstring_to_cstring(""); - FunctionDecl* meth = s->m_methods.at(method_index); - const std::string& ret_type = - qualtype_to_string(meth->getCallResultType(), meth->getASTContext()); - std::cout << " -> RET TYPE: " << ret_type << std::endl; - return cppstring_to_cstring(ret_type); -} - -int cppyy_method_num_args(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) { - return 1; +int cppyy_num_bases(cppyy_type_t type) { + return (int)Cppyy::GetNumBases(type); } -int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) { - return cppyy_method_num_args(handle, method_index); +char* cppyy_base_name(cppyy_type_t type, int base_index){ + return cppstring_to_cstring(Cppyy::GetBaseName (type, base_index)); } -char* cppyy_method_arg_type(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) { - return cppstring_to_cstring("double"); +int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base){ + return (int)Cppyy::IsSubtype( derived, base ); } -char* cppyy_method_arg_default(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) { - return cppstring_to_cstring(""); + +/* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ +ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction) { + return (ptrdiff_t)Cppyy::GetBaseOffset(derived, base, (void*)address, direction, 0); } -char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) { - return cppstring_to_cstring("double"); + +/* method/function reflection information --------------------------------- */ +int cppyy_num_methods(cppyy_scope_t scope) { + return (int)Cppyy::GetNumMethods (scope); } -int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) { - return 0; -} - -cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) { - SimpleScope* s = scope_from_handle(handle); - if (!s) return (cppyy_method_t)0; - return (cppyy_method_t)s->m_methods.at(method_index); +cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth) { + return cppyy_index_t(Cppyy::GetMethodIndexAt (scope, imeth)); } - -/* method properties ----------------------------------------------------- */ -int cppyy_is_constructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { - return 0; +cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name){ +//NEED TO DO: + return (cppyy_index_t*)0; +// return (cppyy_index_t*)Cppyy::GetMethodsFromName(scope, name); } -int cppyy_is_staticmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { - return 1; +char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx) { + TFunction* f = type_get_method(scope, idx); + return cppstring_to_cstring(Cppyy::GetMethodName((Cppyy::TCppMethod_t)f)); } +char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx) { + TFunction* f = type_get_method(scope, idx); + return cppstring_to_cstring(Cppyy::GetMethodResultType((Cppyy::TCppMethod_t)f)); +} + +int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx) { + TFunction* f = type_get_method(scope, idx); + return (int)Cppyy::GetMethodNumArgs((Cppyy::TCppMethod_t)f); +} + +int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx) { + TFunction* f = type_get_method(scope, idx); + return (int)Cppyy::GetMethodReqArgs((Cppyy::TCppMethod_t)f); +} + +char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index) { + TFunction* f = type_get_method(scope, idx); + return cppstring_to_cstring(Cppyy::GetMethodArgType((Cppyy::TCppMethod_t)f, arg_index)); +} + +char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index) { + TFunction* f = type_get_method(scope, idx); + return cppstring_to_cstring(Cppyy::GetMethodArgDefault((Cppyy::TCppMethod_t)f, arg_index)); +} + +char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx) { + return cppstring_to_cstring(Cppyy::GetMethodSignature(scope, idx)); +} + +int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx) { + TFunction* f = type_get_method(scope, idx); + return (int)Cppyy::IsMethodTemplate((Cppyy::TCppMethod_t)f); +} + +int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx) { + return (int)Cppyy::GetMethodNumTemplateArgs(scope, idx); +} + +char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg) { + return cppstring_to_cstring(Cppyy::GetMethodTemplateArgName(scope, idx, iarg)); +} + +cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx) { + return cppyy_method_t(Cppyy::GetMethod(scope, idx)); +} + +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op) { + return cppyy_index_t(Cppyy::GetGlobalOperator(scope, lc, rc, op)); +} + + +/* method properties ------------------------------------------------------ */ +int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx) { + TFunction* f = type_get_method(type, idx); + return (int)Cppyy::IsConstructor((Cppyy::TCppMethod_t)f); +} + +int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx) { + TFunction* f = type_get_method(type, idx); + return (int)Cppyy::IsStaticMethod((Cppyy::TCppMethod_t)f); +} + /* data member reflection information ------------------------------------- */ -int cppyy_num_datamembers(cppyy_scope_t /* handle */) { - return 0; +int cppyy_num_datamembers(cppyy_scope_t scope) { + return (int)Cppyy::GetNumDatamembers(scope); } +char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index) { + return cppstring_to_cstring(Cppyy::GetDatamemberName(scope, datamember_index)); +} + +char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index) { + return cppstring_to_cstring(Cppyy::GetDatamemberType(scope, datamember_index)); +} + +ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index) { + return ptrdiff_t(Cppyy::GetDatamemberOffset(scope, datamember_index)); +} + +int cppyy_datamember_index(cppyy_scope_t scope, const char* name) { + return (int)Cppyy::GetDatamemberIndex(scope, name); +} + + + +/* data member properties ------------------------------------------------- */ +int cppyy_is_publicdata(cppyy_type_t type, int datamember_index) { + return (int)Cppyy::IsPublicData(type, datamember_index); +} + +int cppyy_is_staticdata(cppyy_type_t type, int datamember_index) { + return (int)Cppyy::IsStaticData(type, datamember_index); +} + /* misc helpers ----------------------------------------------------------- */ +RPY_EXTERN +void* cppyy_load_dictionary(const char* lib_name) { + return (void*)(gInterpreter->Load(lib_name) == 0); +} + +long long cppyy_strtoll(const char* str) { + return strtoll(str, NULL, 0); +} + +unsigned long long cppyy_strtoull(const char* str) { + return strtoull(str, NULL, 0); +} + void cppyy_free(void* ptr) { free(ptr); } - -void* cppyy_load_dictionary(const char* lib_name) { - // TODO: need to rethink this; for now it creates reflection info from - // .h while loading lib.so - - // Load a library file in cling's memory. - // if 'system' is true, the library is never unloaded. - // Return 0 on success, -1 on failure. - // R__LOCKGUARD2(gInterpreterMutex); - std::cout << " NOW LOADING: " << lib_name << std::endl; From pypy.commits at gmail.com Tue Jun 28 04:06:14 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 01:06:14 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Clean up the breakpoint system, with a single 'stop_point_break' Message-ID: <57722ff6.571b1c0a.b6c11.2190@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85419:9b947e1ab238 Date: 2016-06-28 10:07 +0200 http://bitbucket.org/pypy/pypy/changeset/9b947e1ab238/ Log: Clean up the breakpoint system, with a single 'stop_point_break' controlled from several internal variables in revdb.c diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -65,6 +65,13 @@ runner(argument) except Exception as e: traceback.print_exc() + print >> sys.stderr + print >> sys.stderr, 'Something went wrong. You are now', + print >> sys.stderr, 'in a pdb; press Ctrl-D to continue.' + import pdb; pdb.post_mortem(sys.exc_info()[2]) + print >> sys.stderr + print >> sys.stderr, 'You are back running %s.' % ( + sys.argv[0],) def command_help(self, argument): """Display commands summary""" diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -8,6 +8,7 @@ CMD_QUIT = -2 # Message(CMD_QUIT) CMD_FORWARD = -3 # Message(CMD_FORWARD, steps, breakpoint_mode) CMD_FUTUREIDS = -4 # Message(CMD_FUTUREIDS, extra=list-of-8bytes-uids) +CMD_PING = -5 # Message(CMD_PING) # extra commands which are not handled by revdb.c, but # by revdb.register_debug_command() CMD_PRINT = 1 # Message(CMD_PRINT, extra=expression) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -139,6 +139,14 @@ Returns the Breakpoint or None. """ assert not self.tainted + + # + currents = self.current_time, self.currently_created_objects + self.send(Message(CMD_PING)) + self.expect_ready() + assert currents == (self.current_time, self.currently_created_objects) + # + self.send(Message(CMD_FORWARD, steps, ord(breakpoint_mode))) # # record the first ANSWER_BREAKPOINT, drop the others @@ -150,7 +158,7 @@ break if bkpt is None: bkpt = Breakpoint(msg.arg1, msg.arg3) - assert msg.cmd == ANSWER_READY + assert msg.cmd == ANSWER_READY, msg self.update_times(msg) return bkpt diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -28,6 +28,10 @@ #define ASYNC_FINALIZER_TRIGGER ((int16_t)0xff46) +#define FID_REGULAR_MODE 'R' +#define FID_SAVED_STATE 'S' +#define FID_JMPBUF_PROTECTED 'J' + typedef struct { Signed version; @@ -40,7 +44,7 @@ rpy_revdb_t rpy_revdb; static char rpy_rev_buffer[16384]; /* max. 32768 */ static int rpy_rev_fileno = -1; -static unsigned char flag_io_disabled; +static char flag_io_disabled = FID_REGULAR_MODE; static void setup_record_mode(int argc, char *argv[]); @@ -199,6 +203,7 @@ They are only invoked when we call GC_invoke_finalizers(), which we only do at stop points in the case of revdb. */ + assert(!RPY_RDB_REPLAY); assert(rpy_revdb.stop_point_break <= rpy_revdb.stop_point_seen + 1); rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1; } @@ -214,26 +219,21 @@ static void record_stop_point(void) { - /* Invoke the finalizers now. This will call boehm_fq_callback(), - which will enqueue the objects in the correct FinalizerQueue. - Then, call boehm_fq_trigger(), which calls finalizer_trigger(). - */ + /* ===== FINALIZERS ===== + + When the GC wants to invoke some finalizers, it causes this + to be called at the stop point. The new-style finalizers + are only enqueued at this point. The old-style finalizers + run immediately, conceptually just *after* the stop point. + */ int i; char *p = rpy_rev_buffer; int64_t done; + /* Write an ASYNC_FINALIZER_TRIGGER packet */ rpy_reverse_db_flush(); + assert(current_packet_size() == 0); - fq_trigger(); - - /* This should be done without emitting anything to the rdb - log. We check that, and emit just a ASYNC_FINALIZER_TRIGGER. - */ - if (current_packet_size() != 0) { - fprintf(stderr, - "record_stop_point emitted unexpected data into the rdb log\n"); - exit(1); - } *(int16_t *)p = ASYNC_FINALIZER_TRIGGER; memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t)); rpy_revdb.buf_p += sizeof(uint64_t); @@ -251,6 +251,10 @@ GC_invoke_finalizers(); in_invoke_finalizers--; RPY_REVDB_EMIT(done = -1;, int64_t _e, done); + + /* Now we're back in normal mode. We trigger the finalizer + queues here. */ + fq_trigger(); } RPY_EXTERN @@ -276,7 +280,7 @@ /* Boehm only */ if (obj->h_hash == 0) { Signed h; - if (flag_io_disabled) { + if (flag_io_disabled != FID_REGULAR_MODE) { /* This is when running debug commands. Don't cache the hash on the object at all. */ return ~((Signed)obj); @@ -378,7 +382,7 @@ r->re_addr = target; r->re_off_prev = 0; - if (!flag_io_disabled) { + if (flag_io_disabled == FID_REGULAR_MODE) { char alive; /* Emit WEAKREF_AFTERWARDS_DEAD, but remember where we emit it. If we deref the weakref and it is still alive, we will patch @@ -415,7 +419,7 @@ { struct WEAKLINK *r = (struct WEAKLINK *)weakref; void *result = r->re_addr; - if (result && !flag_io_disabled) { + if (result && flag_io_disabled == FID_REGULAR_MODE) { char alive; if (!RPY_RDB_REPLAY) { if (r->re_off_prev <= 0) { @@ -471,6 +475,7 @@ #define CMD_QUIT (-2) #define CMD_FORWARD (-3) #define CMD_FUTUREIDS (-4) +#define CMD_PING (-5) #define ANSWER_INIT (-20) #define ANSWER_READY (-21) @@ -482,16 +487,14 @@ static int rpy_rev_sockfd; static const char *rpy_rev_filename; -static uint64_t stopped_time; -static uint64_t stopped_uid; +static uint64_t interactive_break = 1, finalizer_break = -1, uid_break = -1; static uint64_t total_stop_points; -static uint64_t finalizer_trigger_saved_break; static jmp_buf jmp_buf_cancel_execution; static void (*pending_after_forward)(void); static RPyString *empty_string; static uint64_t last_recorded_breakpoint_loc; static int last_recorded_breakpoint_num; -static char breakpoint_mode; +static char breakpoint_mode = 'i'; static uint64_t *future_ids, *future_next_id; static void *finalizer_tree, *destructor_tree; @@ -586,6 +589,15 @@ } } +static void set_revdb_breakpoints(void) +{ + /* note: these are uint64_t, so '-1' is bigger than positive values */ + rpy_revdb.stop_point_break = (interactive_break < finalizer_break ? + interactive_break : finalizer_break); + rpy_revdb.unique_id_break = uid_break; + rpy_revdb.watch_enabled = (breakpoint_mode != 'i'); +} + static void setup_replay_mode(int *argc_p, char **argv_p[]) { int argc = *argc_p; @@ -637,8 +649,9 @@ rpy_revdb.buf_p = rpy_rev_buffer; rpy_revdb.buf_limit = rpy_rev_buffer; rpy_revdb.buf_readend = rpy_rev_buffer; - rpy_revdb.stop_point_break = 1; + rpy_revdb.stop_point_seen = 0; rpy_revdb.unique_id_seen = 1; + set_revdb_breakpoints(); empty_string = make_rpy_string(0); @@ -670,11 +683,15 @@ RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line) { - if (!flag_io_disabled) { + if (flag_io_disabled == FID_REGULAR_MODE) { ssize_t keep; ssize_t full_packet_size; int16_t header; + if (finalizer_break != (uint64_t)-1) { + fprintf(stderr, "reverse_db_fetch: finalizer_break != -1\n"); + exit(1); + } if (rpy_revdb.buf_limit != rpy_revdb.buf_p) { fprintf(stderr, "bad log format: incomplete packet\n"); exit(1); @@ -695,7 +712,8 @@ switch (header) { case ASYNC_FINALIZER_TRIGGER: - if (finalizer_trigger_saved_break != 0) { + //fprintf(stderr, "ASYNC_FINALIZER_TRIGGER\n"); + if (finalizer_break != (uint64_t)-1) { fprintf(stderr, "unexpected multiple " "ASYNC_FINALIZER_TRIGGER\n"); exit(1); @@ -709,10 +727,10 @@ fprintf(stderr, "invalid finalizer break point\n"); exit(1); } - finalizer_trigger_saved_break = rpy_revdb.stop_point_break; - rpy_revdb.stop_point_break = bp; + finalizer_break = bp; + set_revdb_breakpoints(); /* Now we should not fetch anything more until we reach - that finalizer_trigger_saved_break point. */ + that finalizer_break point. */ rpy_revdb.buf_limit = rpy_revdb.buf_p; return; @@ -734,64 +752,94 @@ */ fprintf(stderr, "%s:%d: Attempted to do I/O or access raw memory\n", file, line); - longjmp(jmp_buf_cancel_execution, 1); + if (flag_io_disabled == FID_JMPBUF_PROTECTED) { + longjmp(jmp_buf_cancel_execution, 1); + } + else { + fprintf(stderr, "but we are not in a jmpbuf_protected section\n"); + exit(1); + } } } -static rpy_revdb_t disabled_state; -static void *disabled_exc[2]; +static rpy_revdb_t saved_state; +static void *saved_exc[2]; -static void disable_io(void) +static void change_flag_io_disabled(char oldval, char newval) { - if (flag_io_disabled) { - fprintf(stderr, "unexpected recursive disable_io()\n"); + if (flag_io_disabled != oldval) { + fprintf(stderr, "change_flag_io_disabled(%c, %c) but got %c\n", + oldval, newval, flag_io_disabled); exit(1); } - disabled_state = rpy_revdb; /* save the complete struct */ - disabled_exc[0] = pypy_g_ExcData.ed_exc_type; - disabled_exc[1] = pypy_g_ExcData.ed_exc_value; + flag_io_disabled = newval; +} + +static void save_state(void) +{ + /* The program is switching from replaying execution to + time-paused mode. In time-paused mode, we can run more + app-level code like watch points or interactive prints, + but they must not be matched against the log, and they must + not involve generic I/O. + */ + change_flag_io_disabled(FID_REGULAR_MODE, FID_SAVED_STATE); + + saved_state = rpy_revdb; /* save the complete struct */ + + rpy_revdb.unique_id_seen = (-1ULL) << 63; + rpy_revdb.watch_enabled = 0; + rpy_revdb.stop_point_break = -1; + rpy_revdb.unique_id_break = -1; + rpy_revdb.buf_p = rpy_rev_buffer; /* anything readable */ + rpy_revdb.buf_limit = rpy_rev_buffer; /* same as buf_p */ +} + +static void restore_state(void) +{ + /* The program is switching from time-paused mode to replaying + execution. */ + change_flag_io_disabled(FID_SAVED_STATE, FID_REGULAR_MODE); + + /* restore the complete struct */ + rpy_revdb = saved_state; + + /* set the breakpoint fields to the current value of the *_break + global variables, which may be different from what is in + 'save_state' */ + set_revdb_breakpoints(); +} + +static void protect_jmpbuf(void) +{ + change_flag_io_disabled(FID_SAVED_STATE, FID_JMPBUF_PROTECTED); + saved_exc[0] = pypy_g_ExcData.ed_exc_type; + saved_exc[1] = pypy_g_ExcData.ed_exc_value; pypy_g_ExcData.ed_exc_type = NULL; pypy_g_ExcData.ed_exc_value = NULL; - rpy_revdb.buf_p = rpy_rev_buffer; /* anything readable */ - rpy_revdb.buf_limit = rpy_rev_buffer; /* same as buf_p */ - flag_io_disabled = 1; } -static void enable_io(int restore_breaks) +static void unprotect_jmpbuf(void) { - uint64_t v1, v2; - if (!flag_io_disabled) { - fprintf(stderr, "unexpected enable_io()\n"); - exit(1); - } - flag_io_disabled = 0; - + change_flag_io_disabled(FID_JMPBUF_PROTECTED, FID_SAVED_STATE); if (pypy_g_ExcData.ed_exc_type != NULL) { fprintf(stderr, "Command crashed with %.*s\n", (int)(pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.length), pypy_g_ExcData.ed_exc_type->ov_name->rs_chars.items); exit(1); } - /* restore the complete struct, with the exception of '*_break' */ - v1 = rpy_revdb.stop_point_break; - v2 = rpy_revdb.unique_id_break; - rpy_revdb = disabled_state; - if (!restore_breaks) { - rpy_revdb.stop_point_break = v1; - rpy_revdb.unique_id_break = v2; - } - pypy_g_ExcData.ed_exc_type = disabled_exc[0]; - pypy_g_ExcData.ed_exc_value = disabled_exc[1]; + pypy_g_ExcData.ed_exc_type = saved_exc[0]; + pypy_g_ExcData.ed_exc_value = saved_exc[1]; } static void execute_rpy_function(rpy_revdb_command_fn func, rpy_revdb_command_t *cmd, RPyString *extra) { - disable_io(); + protect_jmpbuf(); if (setjmp(jmp_buf_cancel_execution) == 0) func(cmd, extra); - enable_io(0); + unprotect_jmpbuf(); } static void check_at_end(uint64_t stop_points) @@ -888,13 +936,13 @@ fprintf(stderr, "CMD_FORWARD: negative step\n"); exit(1); } - rpy_revdb.stop_point_break = stopped_time + cmd->arg1; + assert(flag_io_disabled == FID_SAVED_STATE); + interactive_break = saved_state.stop_point_seen + cmd->arg1; breakpoint_mode = (char)cmd->arg2; if (breakpoint_mode == 'r') { last_recorded_breakpoint_loc = 0; pending_after_forward = &answer_recorded_breakpoint; } - rpy_revdb.watch_enabled = (breakpoint_mode != 'i'); } static void command_future_ids(rpy_revdb_command_t *cmd, char *extra) @@ -902,7 +950,7 @@ free(future_ids); if (cmd->extra_size == 0) { future_ids = NULL; - rpy_revdb.unique_id_break = 0; + uid_break = 0; } else { assert(cmd->extra_size % sizeof(uint64_t) == 0); @@ -914,7 +962,7 @@ } memcpy(future_ids, extra, cmd->extra_size); future_ids[cmd->extra_size / sizeof(uint64_t)] = 0; - rpy_revdb.unique_id_break = *future_ids; + uid_break = *future_ids; } future_next_id = future_ids; } @@ -940,41 +988,16 @@ execute_rpy_function(rpy_revdb_commands.rp_funcs[i], cmd, s); } -static void save_state(void) -{ - stopped_time = rpy_revdb.stop_point_seen; - stopped_uid = rpy_revdb.unique_id_seen; - rpy_revdb.unique_id_seen = (-1ULL) << 63; - rpy_revdb.watch_enabled = 0; -} - -static void restore_state(void) -{ - rpy_revdb.stop_point_seen = stopped_time; - rpy_revdb.unique_id_seen = stopped_uid; - stopped_time = 0; - stopped_uid = 0; -} - RPY_EXTERN void rpy_reverse_db_watch_save_state(void) { - if (stopped_time != 0) { - fprintf(stderr, "unexpected recursive watch_save_state\n"); - exit(1); - } save_state(); - disable_io(); - rpy_revdb.stop_point_break = 0; - rpy_revdb.unique_id_break = 0; } RPY_EXTERN void rpy_reverse_db_watch_restore_state(bool_t any_watch_point) { - enable_io(1); restore_state(); - assert(!rpy_revdb.watch_enabled); rpy_revdb.watch_enabled = any_watch_point; } @@ -982,12 +1005,17 @@ static void replay_stop_point(void) { - if (finalizer_trigger_saved_break != 0) + if (finalizer_break != (uint64_t)-1) replay_call_destructors(); + if (rpy_revdb.stop_point_break != interactive_break) { + fprintf(stderr, "mismatch between interactive_break and " + "stop_point_break\n"); + exit(1); + } + while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) { save_state(); - breakpoint_mode = 0; if (pending_after_forward) { void (*fn)(void) = pending_after_forward; @@ -996,7 +1024,10 @@ } else { rpy_revdb_command_t cmd; - write_answer(ANSWER_READY, stopped_time, stopped_uid, 0); + write_answer(ANSWER_READY, + saved_state.stop_point_seen, + saved_state.unique_id_seen, + 0); read_sock(&cmd, sizeof(cmd)); char extra[cmd.extra_size + 1]; @@ -1022,6 +1053,9 @@ command_future_ids(&cmd, extra); break; + case CMD_PING: /* to get only the ANSWER_READY */ + break; + default: command_default(&cmd, extra); break; @@ -1054,7 +1088,7 @@ RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num) { - if (stopped_time != 0) { + if (flag_io_disabled != FID_REGULAR_MODE) { fprintf(stderr, "revdb.breakpoint(): cannot be called from a " "debug command\n"); exit(1); @@ -1069,9 +1103,16 @@ last_recorded_breakpoint_num = num; return; - default: /* 'b' or '\0' for default handling of breakpoints */ - rpy_revdb.stop_point_break = rpy_revdb.stop_point_seen + 1; + case 'b': /* default handling of breakpoints */ + interactive_break = rpy_revdb.stop_point_seen + 1; + set_revdb_breakpoints(); write_answer(ANSWER_BREAKPOINT, rpy_revdb.stop_point_break, 0, num); + return; + + default: + fprintf(stderr, "bad value %d of breakpoint_mode\n", + (int)breakpoint_mode); + exit(1); } } @@ -1080,13 +1121,17 @@ { switch (value_id) { case 'c': /* current_time() */ - return stopped_time ? stopped_time : rpy_revdb.stop_point_seen; + return (flag_io_disabled == FID_REGULAR_MODE ? + rpy_revdb.stop_point_seen : + saved_state.stop_point_seen); case 't': /* total_time() */ return total_stop_points; case 'b': /* current_break_time() */ - return rpy_revdb.stop_point_break; + return interactive_break; case 'u': /* currently_created_objects() */ - return stopped_uid ? stopped_uid : rpy_revdb.unique_id_seen; + return (flag_io_disabled == FID_REGULAR_MODE ? + rpy_revdb.unique_id_seen : + saved_state.unique_id_seen); default: return -1; } @@ -1096,18 +1141,23 @@ uint64_t rpy_reverse_db_unique_id_break(void *new_object) { uint64_t uid = rpy_revdb.unique_id_seen; + bool_t watch_enabled = rpy_revdb.watch_enabled; + if (!new_object) { fprintf(stderr, "out of memory: allocation failed, cannot continue\n"); exit(1); } + + save_state(); if (rpy_revdb_commands.rp_alloc) { - bool_t watch_enabled = rpy_revdb.watch_enabled; - rpy_reverse_db_watch_save_state(); + protect_jmpbuf(); if (setjmp(jmp_buf_cancel_execution) == 0) rpy_revdb_commands.rp_alloc(uid, new_object); - rpy_reverse_db_watch_restore_state(watch_enabled); + unprotect_jmpbuf(); } - rpy_revdb.unique_id_break = *future_next_id++; + uid_break = *future_next_id++; + restore_state(); + rpy_revdb.watch_enabled = watch_enabled; return uid; } @@ -1221,14 +1271,15 @@ static void replay_call_destructors(void) { - rpy_revdb.stop_point_break = finalizer_trigger_saved_break; - finalizer_trigger_saved_break = 0; fq_trigger(); /* Re-enable fetching (disabled when we saw ASYNC_FINALIZER_TRIGGER), and fetch the uid's of dying objects with old-style destructors. */ + finalizer_break = -1; + set_revdb_breakpoints(); rpy_reverse_db_fetch(__FILE__, __LINE__); + while (1) { int64_t uid; struct destructor_s d_dummy, *entry, **item; diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -327,3 +327,9 @@ child = self.replay() child.send(Message(CMD_FORWARD, 3001)) child.expect(ANSWER_AT_END) + + def test_bug1(self): + child = self.replay() + for i in range(50): + child.send(Message(CMD_FORWARD, i)) + child.expect_ready() From pypy.commits at gmail.com Tue Jun 28 05:10:31 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 02:10:31 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Bug fix Message-ID: <57723f07.49c51c0a.b8da6.ffffd85a@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85420:0cc665a2fb4e Date: 2016-06-28 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/0cc665a2fb4e/ Log: Bug fix diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -3,7 +3,7 @@ import traceback from contextlib import contextmanager -from rpython.translator.revdb.process import ReplayProcessGroup, maxint64 +from rpython.translator.revdb.process import ReplayProcessGroup from rpython.translator.revdb.process import Breakpoint r_cmdline = re.compile(r"(\S+)\s*(.*)") diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -3,9 +3,6 @@ from rpython.translator.revdb.message import * -maxint64 = int(2**63 - 1) - - class Breakpoint(Exception): def __init__(self, time, num): self.time = time @@ -24,7 +21,9 @@ self.stack_depth = 0 # breaks if the depth becomes lower than this def __repr__(self): - return 'AllBreakpoints(%r, %d)' % (self.num2name, self.stack_depth) + return 'AllBreakpoints(%r, %r, %r, %d)' % ( + self.num2name, self.watchvalues, self.watchuids, + self.stack_depth) def compare(self, other): if (self.num2name == other.num2name and @@ -334,6 +333,8 @@ if self.all_breakpoints.watchuids: uids = set() uids.update(*self.all_breakpoints.watchuids.values()) + #print self.all_breakpoints + #print '\t===>', uids self.attach_printed_objects(uids, watch_env=True) def update_breakpoints(self): @@ -379,6 +380,7 @@ if name.startswith('W'): _, text = self.check_watchpoint_expr(name[1:]) if text != self.all_breakpoints.watchvalues[num]: + #print self.active.pid print 'updating watchpoint value: %s => %s' % ( name[1:], text) self.all_breakpoints.watchvalues[num] = text @@ -454,6 +456,10 @@ future_uids.sort() pack_uids = [struct.pack('q', uid) for uid in future_uids] pack_uids = ''.join(pack_uids) + #print '%d: from %d: CMD_FUTUREIDS %r' % ( + # self.active.pid, + # self.active.current_time, + # future_uids) self.active.send(Message(CMD_FUTUREIDS, extra=pack_uids)) self.active.expect_ready() self.active.printed_objects = ( @@ -480,6 +486,8 @@ def attach_printed_objects(self, uids, watch_env): for uid in uids: nid = self.all_printed_objects[uid] + #print '%d: %s => %s (watch_env=%d)' % (self.active.pid, nid, uid, + # watch_env) self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env))) self.active.expect_ready() diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1155,7 +1155,7 @@ rpy_revdb_commands.rp_alloc(uid, new_object); unprotect_jmpbuf(); } - uid_break = *future_next_id++; + uid_break = *++future_next_id; restore_state(); rpy_revdb.watch_enabled = watch_enabled; return uid; From pypy.commits at gmail.com Tue Jun 28 07:21:47 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 04:21:47 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: started pypy support: recognize in pyparser the syntax of a Message-ID: <57725dcb.c255c20a.1add.ffffdae0@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85421:91e8176fb344 Date: 2016-06-28 13:23 +0200 http://bitbucket.org/pypy/pypy/changeset/91e8176fb344/ Log: started pypy support: recognize in pyparser the syntax of a dollar sign followed by a decimal number diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7 --- a/pypy/interpreter/pyparser/data/Grammar2.7 +++ b/pypy/interpreter/pyparser/data/Grammar2.7 @@ -104,7 +104,7 @@ '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+) + NAME | NUMBER | STRING+ | revdb_metavar) listmaker: test ( list_for | (',' test)* [','] ) testlist_comp: test ( comp_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test @@ -141,3 +141,5 @@ encoding_decl: NAME yield_expr: 'yield' [testlist] + +revdb_metavar: '$NUM' diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -140,7 +140,10 @@ special = group(states, makeEOL(), groupStr(states, "@:;.,`")) - funny = group(states, operator, bracket, special) + revdb_metavar = chain(states, + groupStr(states, "$"), + atleastonce(states, makeDigits())) + funny = group(states, operator, bracket, special, revdb_metavar) # ____________________________________________________________ def makeStrPrefix (): return chain(states, diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py --- a/pypy/interpreter/pyparser/pytoken.py +++ b/pypy/interpreter/pyparser/pytoken.py @@ -67,5 +67,6 @@ # extra PyPy-specific tokens _add_tok("COMMENT") _add_tok("NL") +_add_tok("REVDBMETAVAR", "$NUM") del _add_tok diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -25,37 +25,38 @@ accepts = [True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, False, - False, False, True, False, False, True, False, - False, True, False, True, False, True, False, - False, True, False, False, True, True, True, - False, False, True, False, False, False, True] + False, False, False, True, False, False, True, + False, False, True, False, True, True, False, + True, False, False, True, False, False, True, + True, True, False, False, True, False, False, + False, True] states = [ # 0 {'\t': 0, '\n': 13, '\x0c': 0, - '\r': 14, ' ': 0, '!': 10, '"': 16, - '#': 18, '%': 12, '&': 12, "'": 15, - '(': 13, ')': 13, '*': 7, '+': 12, - ',': 13, '-': 12, '.': 6, '/': 11, - '0': 4, '1': 5, '2': 5, '3': 5, - '4': 5, '5': 5, '6': 5, '7': 5, - '8': 5, '9': 5, ':': 13, ';': 13, - '<': 9, '=': 12, '>': 8, '@': 13, - 'A': 1, 'B': 2, 'C': 1, 'D': 1, - 'E': 1, 'F': 1, 'G': 1, 'H': 1, - 'I': 1, 'J': 1, 'K': 1, 'L': 1, - 'M': 1, 'N': 1, 'O': 1, 'P': 1, - 'Q': 1, 'R': 3, 'S': 1, 'T': 1, - 'U': 2, 'V': 1, 'W': 1, 'X': 1, - 'Y': 1, 'Z': 1, '[': 13, '\\': 17, - ']': 13, '^': 12, '_': 1, '`': 13, - 'a': 1, 'b': 2, 'c': 1, 'd': 1, - 'e': 1, 'f': 1, 'g': 1, 'h': 1, - 'i': 1, 'j': 1, 'k': 1, 'l': 1, - 'm': 1, 'n': 1, 'o': 1, 'p': 1, - 'q': 1, 'r': 3, 's': 1, 't': 1, - 'u': 2, 'v': 1, 'w': 1, 'x': 1, - 'y': 1, 'z': 1, '{': 13, '|': 12, - '}': 13, '~': 13}, + '\r': 14, ' ': 0, '!': 10, '"': 17, + '#': 19, '$': 15, '%': 12, '&': 12, + "'": 16, '(': 13, ')': 13, '*': 7, + '+': 12, ',': 13, '-': 12, '.': 6, + '/': 11, '0': 4, '1': 5, '2': 5, + '3': 5, '4': 5, '5': 5, '6': 5, + '7': 5, '8': 5, '9': 5, ':': 13, + ';': 13, '<': 9, '=': 12, '>': 8, + '@': 13, 'A': 1, 'B': 2, 'C': 1, + 'D': 1, 'E': 1, 'F': 1, 'G': 1, + 'H': 1, 'I': 1, 'J': 1, 'K': 1, + 'L': 1, 'M': 1, 'N': 1, 'O': 1, + 'P': 1, 'Q': 1, 'R': 3, 'S': 1, + 'T': 1, 'U': 2, 'V': 1, 'W': 1, + 'X': 1, 'Y': 1, 'Z': 1, '[': 13, + '\\': 18, ']': 13, '^': 12, '_': 1, + '`': 13, 'a': 1, 'b': 2, 'c': 1, + 'd': 1, 'e': 1, 'f': 1, 'g': 1, + 'h': 1, 'i': 1, 'j': 1, 'k': 1, + 'l': 1, 'm': 1, 'n': 1, 'o': 1, + 'p': 1, 'q': 1, 'r': 3, 's': 1, + 't': 1, 'u': 2, 'v': 1, 'w': 1, + 'x': 1, 'y': 1, 'z': 1, '{': 13, + '|': 12, '}': 13, '~': 13}, # 1 {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, @@ -74,7 +75,7 @@ 't': 1, 'u': 1, 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1}, # 2 - {'"': 16, "'": 15, '0': 1, '1': 1, + {'"': 17, "'": 16, '0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1, 'A': 1, 'B': 1, 'C': 1, 'D': 1, @@ -92,7 +93,7 @@ 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1}, # 3 - {'"': 16, "'": 15, '0': 1, '1': 1, + {'"': 17, "'": 16, '0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1, 'A': 1, 'B': 1, 'C': 1, 'D': 1, @@ -110,22 +111,22 @@ 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1}, # 4 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'B': 22, - 'E': 25, 'J': 13, 'L': 13, 'O': 20, - 'X': 19, 'b': 22, 'e': 25, 'j': 13, - 'l': 13, 'o': 20, 'x': 19}, + {'.': 25, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 24, '9': 24, 'B': 23, + 'E': 26, 'J': 13, 'L': 13, 'O': 21, + 'X': 20, 'b': 23, 'e': 26, 'j': 13, + 'l': 13, 'o': 21, 'x': 20}, # 5 - {'.': 24, '0': 5, '1': 5, '2': 5, + {'.': 25, '0': 5, '1': 5, '2': 5, '3': 5, '4': 5, '5': 5, '6': 5, - '7': 5, '8': 5, '9': 5, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, + '7': 5, '8': 5, '9': 5, 'E': 26, + 'J': 13, 'L': 13, 'e': 26, 'j': 13, 'l': 13}, # 6 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26}, + {'0': 27, '1': 27, '2': 27, '3': 27, + '4': 27, '5': 27, '6': 27, '7': 27, + '8': 27, '9': 27}, # 7 {'*': 12, '=': 13}, # 8 @@ -143,107 +144,115 @@ # 14 {'\n': 13}, # 15 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 28, '\\': 29}, + {'0': 28, '1': 28, '2': 28, '3': 28, + '4': 28, '5': 28, '6': 28, '7': 28, + '8': 28, '9': 28}, # 16 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 31, '\\': 32}, + {automata.DEFAULT: 32, '\n': 29, + '\r': 29, "'": 30, '\\': 31}, # 17 + {automata.DEFAULT: 35, '\n': 29, + '\r': 29, '"': 33, '\\': 34}, + # 18 {'\n': 13, '\r': 14}, - # 18 - {automata.DEFAULT: 18, '\n': 27, '\r': 27}, # 19 - {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'a': 34, 'b': 34, 'c': 34, 'd': 34, - 'e': 34, 'f': 34}, + {automata.DEFAULT: 19, '\n': 29, '\r': 29}, # 20 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35}, + {'0': 36, '1': 36, '2': 36, '3': 36, + '4': 36, '5': 36, '6': 36, '7': 36, + '8': 36, '9': 36, 'A': 36, 'B': 36, + 'C': 36, 'D': 36, 'E': 36, 'F': 36, + 'a': 36, 'b': 36, 'c': 36, 'd': 36, + 'e': 36, 'f': 36}, # 21 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, + {'0': 37, '1': 37, '2': 37, '3': 37, + '4': 37, '5': 37, '6': 37, '7': 37}, + # 22 + {'.': 25, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 24, '9': 24, 'E': 26, + 'J': 13, 'L': 13, 'e': 26, 'j': 13, 'l': 13}, - # 22 - {'0': 36, '1': 36}, # 23 - {'.': 24, '0': 23, '1': 23, '2': 23, - '3': 23, '4': 23, '5': 23, '6': 23, - '7': 23, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'e': 25, 'j': 13}, + {'0': 38, '1': 38}, # 24 - {'0': 24, '1': 24, '2': 24, '3': 24, - '4': 24, '5': 24, '6': 24, '7': 24, - '8': 24, '9': 24, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, + {'.': 25, '0': 24, '1': 24, '2': 24, + '3': 24, '4': 24, '5': 24, '6': 24, + '7': 24, '8': 24, '9': 24, 'E': 26, + 'J': 13, 'e': 26, 'j': 13}, # 25 - {'+': 38, '-': 38, '0': 39, '1': 39, - '2': 39, '3': 39, '4': 39, '5': 39, - '6': 39, '7': 39, '8': 39, '9': 39}, + {'0': 25, '1': 25, '2': 25, '3': 25, + '4': 25, '5': 25, '6': 25, '7': 25, + '8': 25, '9': 25, 'E': 39, 'J': 13, + 'e': 39, 'j': 13}, # 26 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, + {'+': 40, '-': 40, '0': 41, '1': 41, + '2': 41, '3': 41, '4': 41, '5': 41, + '6': 41, '7': 41, '8': 41, '9': 41}, # 27 + {'0': 27, '1': 27, '2': 27, '3': 27, + '4': 27, '5': 27, '6': 27, '7': 27, + '8': 27, '9': 27, 'E': 39, 'J': 13, + 'e': 39, 'j': 13}, + # 28 + {'0': 28, '1': 28, '2': 28, '3': 28, + '4': 28, '5': 28, '6': 28, '7': 28, + '8': 28, '9': 28}, + # 29 {}, - # 28 + # 30 {"'": 13}, - # 29 - {automata.DEFAULT: 40, '\n': 13, '\r': 14}, - # 30 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, # 31 + {automata.DEFAULT: 42, '\n': 13, '\r': 14}, + # 32 + {automata.DEFAULT: 32, '\n': 29, + '\r': 29, "'": 13, '\\': 31}, + # 33 {'"': 13}, - # 32 - {automata.DEFAULT: 41, '\n': 13, '\r': 14}, - # 33 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, # 34 - {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'L': 13, 'a': 34, 'b': 34, 'c': 34, - 'd': 34, 'e': 34, 'f': 34, 'l': 13}, + {automata.DEFAULT: 43, '\n': 13, '\r': 14}, # 35 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35, + {automata.DEFAULT: 35, '\n': 29, + '\r': 29, '"': 13, '\\': 34}, + # 36 + {'0': 36, '1': 36, '2': 36, '3': 36, + '4': 36, '5': 36, '6': 36, '7': 36, + '8': 36, '9': 36, 'A': 36, 'B': 36, + 'C': 36, 'D': 36, 'E': 36, 'F': 36, + 'L': 13, 'a': 36, 'b': 36, 'c': 36, + 'd': 36, 'e': 36, 'f': 36, 'l': 13}, + # 37 + {'0': 37, '1': 37, '2': 37, '3': 37, + '4': 37, '5': 37, '6': 37, '7': 37, 'L': 13, 'l': 13}, - # 36 - {'0': 36, '1': 36, 'L': 13, 'l': 13}, - # 37 - {'+': 42, '-': 42, '0': 43, '1': 43, - '2': 43, '3': 43, '4': 43, '5': 43, - '6': 43, '7': 43, '8': 43, '9': 43}, # 38 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39}, + {'0': 38, '1': 38, 'L': 13, 'l': 13}, # 39 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39, 'J': 13, 'j': 13}, + {'+': 44, '-': 44, '0': 45, '1': 45, + '2': 45, '3': 45, '4': 45, '5': 45, + '6': 45, '7': 45, '8': 45, '9': 45}, # 40 - {automata.DEFAULT: 40, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '8': 41, '9': 41}, # 41 - {automata.DEFAULT: 41, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '8': 41, '9': 41, 'J': 13, 'j': 13}, # 42 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43}, + {automata.DEFAULT: 42, '\n': 29, + '\r': 29, "'": 13, '\\': 31}, # 43 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43, 'J': 13, 'j': 13}, + {automata.DEFAULT: 43, '\n': 29, + '\r': 29, '"': 13, '\\': 34}, + # 44 + {'0': 45, '1': 45, '2': 45, '3': 45, + '4': 45, '5': 45, '6': 45, '7': 45, + '8': 45, '9': 45}, + # 45 + {'0': 45, '1': 45, '2': 45, '3': 45, + '4': 45, '5': 45, '6': 45, '7': 45, + '8': 45, '9': 45, 'J': 13, 'j': 13}, ] pseudoDFA = automata.DFA(states, accepts) diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -220,6 +220,10 @@ last_comment = '' elif initial == '\\': # continued stmt continued = 1 + elif initial == '$': + token_list.append((tokens.REVDBMETAVAR, token, + lnum, start, line)) + last_comment = '' else: if initial in '([{': if parenlev == 0: diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -165,3 +165,12 @@ for linefeed in ["\r\n","\r"]: tree = self.parse(fmt % linefeed) assert expected_tree == tree + + def test_revdb_dollar_num(self): + self.parse('$0') + self.parse('$5') + self.parse('$42') + self.parse('2+$42.attrname') + py.test.raises(SyntaxError, self.parse, '$') + py.test.raises(SyntaxError, self.parse, '$a') + py.test.raises(SyntaxError, self.parse, '$.5') From pypy.commits at gmail.com Tue Jun 28 07:31:32 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 04:31:32 -0700 (PDT) Subject: [pypy-commit] pypy default: ast.py was manually edited (in a non-essential way), fix asdl_py.py to match Message-ID: <57726014.c7aec20a.d0cfb.ffff8303@mx.google.com> Author: Armin Rigo Branch: Changeset: r85422:00804b502576 Date: 2016-06-28 13:32 +0200 http://bitbucket.org/pypy/pypy/changeset/00804b502576/ Log: ast.py was manually edited (in a non-essential way), fix asdl_py.py to match diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -400,7 +400,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): From pypy.commits at gmail.com Tue Jun 28 08:02:37 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 05:02:37 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: astcompiler and basic interpreter support for $NUM Message-ID: <5772675d.88c11c0a.7d12e.ffff8e9e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85423:0d91eaae941b Date: 2016-06-28 14:03 +0200 http://bitbucket.org/pypy/pypy/changeset/0d91eaae941b/ Log: astcompiler and basic interpreter support for $NUM diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py --- a/lib-python/2.7/opcode.py +++ b/lib-python/2.7/opcode.py @@ -194,5 +194,6 @@ def_op('CALL_METHOD', 202) # #args not including 'self' def_op('BUILD_LIST_FROM_ARG', 203) jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements +def_op('LOAD_REVDB_VAR', 205) # reverse debugger (syntax example: $5) del def_op, name_op, jrel_op, jabs_op diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -664,6 +664,7 @@ ops.JUMP_IF_NOT_DEBUG: 0, ops.BUILD_LIST_FROM_ARG: 1, + ops.LOAD_REVDB_VAR: 1, } diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -1534,6 +1534,8 @@ return Num.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Str): return Str.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_RevDBMetaVar): + return RevDBMetaVar.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Attribute): return Attribute.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Subscript): @@ -2344,6 +2346,41 @@ State.ast_type('Str', 'expr', ['s']) +class RevDBMetaVar(expr): + + def __init__(self, metavar, lineno, col_offset): + self.metavar = metavar + expr.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_RevDBMetaVar(self) + + def mutate_over(self, visitor): + return visitor.visit_RevDBMetaVar(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_RevDBMetaVar) + w_metavar = space.wrap(self.metavar) # int + space.setattr(w_node, space.wrap('metavar'), w_metavar) + w_lineno = space.wrap(self.lineno) # int + space.setattr(w_node, space.wrap('lineno'), w_lineno) + w_col_offset = space.wrap(self.col_offset) # int + space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_metavar = get_field(space, w_node, 'metavar', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _metavar = space.int_w(w_metavar) + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return RevDBMetaVar(_metavar, _lineno, _col_offset) + +State.ast_type('RevDBMetaVar', 'expr', ['metavar']) + + class Attribute(expr): def __init__(self, value, attr, ctx, lineno, col_offset): @@ -3439,6 +3476,8 @@ return self.default_visitor(node) def visit_Str(self, node): return self.default_visitor(node) + def visit_RevDBMetaVar(self, node): + return self.default_visitor(node) def visit_Attribute(self, node): return self.default_visitor(node) def visit_Subscript(self, node): @@ -3655,6 +3694,9 @@ def visit_Str(self, node): pass + def visit_RevDBMetaVar(self, node): + pass + def visit_Attribute(self, node): node.value.walkabout(self) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1160,6 +1160,11 @@ elif first_child_type == tokens.BACKQUOTE: expr = self.handle_testlist(atom_node.get_child(1)) return ast.Repr(expr, atom_node.get_lineno(), atom_node.get_column()) + elif first_child_type == tokens.REVDBMETAVAR: + string = atom_node.get_child(0).get_value() + return ast.RevDBMetaVar(int(string[1:]), + atom_node.get_lineno(), + atom_node.get_column()) else: raise AssertionError("unknown atom") diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1196,6 +1196,14 @@ sub.value.walkabout(self) self._compile_slice(sub.slice, sub.ctx) + def visit_RevDBMetaVar(self, node): + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import dbstate + if dbstate.extend_syntax_with_dollar_num: + self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar) + return + self.error("$NUM is only valid in the reverse-debugger", node) + class TopLevelCodeGenerator(PythonCodeGenerator): diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -940,6 +940,15 @@ yield (self.st, "x=(lambda: (-0.0, 0.0), lambda: (0.0, -0.0))[1]()", 'repr(x)', '(0.0, -0.0)') + def test_revdb_metavar(self): + from pypy.interpreter.reverse_debugging import dbstate, setup_revdb + self.space.config.translation.reverse_debugger = True + setup_revdb(self.space) + dbstate.extend_syntax_with_dollar_num = True + dbstate.metavars = [self.space.wrap(6)] + self.simple_test("x = 7*$0", "x", 42) + dbstate.extend_syntax_with_dollar_num = False + class AppTestCompiler: diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -71,6 +71,7 @@ | Repr(expr value) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? + | RevDBMetaVar(int metavar) -- other literals? bools? -- the following expression can appear in assignment context diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -400,7 +400,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -412,6 +412,10 @@ def startup(self): # To be called before using the space + if self.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import setup_revdb + setup_revdb(self) + self.threadlocals.enter_thread(self) # Initialize already imported builtin modules diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -436,6 +436,8 @@ self.WITH_CLEANUP(oparg, next_instr) elif opcode == opcodedesc.YIELD_VALUE.index: self.YIELD_VALUE(oparg, next_instr) + elif opcode == opcodedesc.LOAD_REVDB_VAR.index: + self.LOAD_REVDB_VAR(oparg, next_instr) else: self.MISSING_OPCODE(oparg, next_instr) @@ -1305,6 +1307,11 @@ w_dict = self.peekvalue() self.space.setitem(w_dict, w_key, w_value) + def LOAD_REVDB_VAR(self, oparg, next_instr): + from pypy.interpreter.reverse_debugging import load_metavar + w_var = load_metavar(oparg) + self.pushvalue(w_var) + ### ____________________________________________________________ ### diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7 --- a/pypy/interpreter/pyparser/data/Grammar2.7 +++ b/pypy/interpreter/pyparser/data/Grammar2.7 @@ -104,7 +104,7 @@ '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+ | revdb_metavar) + NAME | NUMBER | STRING+ | '$NUM') listmaker: test ( list_for | (',' test)* [','] ) testlist_comp: test ( comp_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test @@ -141,5 +141,3 @@ encoding_decl: NAME yield_expr: 'yield' [testlist] - -revdb_metavar: '$NUM' diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/reverse_debugging.py @@ -0,0 +1,41 @@ +import sys +from rpython.rlib import revdb +from rpython.rlib.debug import make_sure_not_resized +from pypy.interpreter.error import oefmt + + +class DBState: + extend_syntax_with_dollar_num = False + metavars = [] + +dbstate = DBState() + + +def setup_revdb(space): + assert space.config.translation.reverse_debugger + dbstate.space = space + #make_sure_not_resized(dbstate.breakpoint_funcnames) + #make_sure_not_resized(dbstate.watch_progs) + make_sure_not_resized(dbstate.metavars) + #revdb.register_debug_command(revdb.CMD_PRINT, lambda_print) + #revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace) + #revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals) + #revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints) + #revdb.register_debug_command(revdb.CMD_MOREINFO, lambda_moreinfo) + #revdb.register_debug_command("ALLOCATING", lambda_allocating) + #revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid) + #revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch) + #revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) + +def load_metavar(oparg): + space = dbstate.space + metavars = dbstate.metavars + w_var = metavars[oparg] if oparg < len(metavars) else None + if w_var is None: + raise oefmt(space.w_NameError, "no constant object '$%d'", + oparg) + if w_var is space.w_Ellipsis: + raise oefmt(space.w_RuntimeError, + "'$%d' refers to an object created later in time", + oparg) + return w_var From pypy.commits at gmail.com Tue Jun 28 08:17:56 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 28 Jun 2016 05:17:56 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: removed & renamed load/store combinations that are rewritten to vec_load/vec_store Message-ID: <57726af4.8999c20a.3860d.5388@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85424:c3391ca232ac Date: 2016-06-27 16:07 +0200 http://bitbucket.org/pypy/pypy/changeset/c3391ca232ac/ Log: removed & renamed load/store combinations that are rewritten to vec_load/vec_store diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -158,10 +158,10 @@ def _emit_mul_if_factor_offset_not_supported(self, index_box, factor, offset): - factor, offset, index_box = cpu_simplify_scale(self.cpu, indexbox, factor, offset) - if index_box: + factor, offset, new_index_box = cpu_simplify_scale(self.cpu, index_box, factor, offset) + if index_box is not new_index_box: self.emit_op(index_box) - return factor, offset, index_box + return factor, offset, new_index_box def emit_gc_load_or_indexed(self, op, ptr_box, index_box, itemsize, factor, offset, sign, type='i'): diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py --- a/rpython/jit/backend/llsupport/vector_ext.py +++ b/rpython/jit/backend/llsupport/vector_ext.py @@ -3,6 +3,7 @@ from rpython.rlib.objectmodel import specialize, always_inline from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT) from rpython.jit.metainterp.resoperation import rop +from rpython.jit.metainterp.optimizeopt.schedule import forwarded_vecinfo class TypeRestrict(object): ANY_TYPE = '\x00' diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -26,22 +26,7 @@ class VectorAssembler(object): _mixin_ = True - def _emit_getitem(self, op, arglocs, regalloc): - # prepares item scale (raw_load does not) - resloc, base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs - scale = get_scale(size_loc.value) - xxx - src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale) - self._vec_load(resloc, src_addr, integer_loc.value, - size_loc.value, aligned_loc.value) - - emit_vec_getarrayitem_raw_i = _emit_getitem - emit_vec_getarrayitem_raw_f = _emit_getitem - - emit_vec_getarrayitem_gc_i = _emit_getitem - emit_vec_getarrayitem_gc_f = _emit_getitem - - def emit_vec_raw_load_f(self, op, arglocs, regalloc): + def emit_vec_load_f(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, integer_loc, aligned_loc = arglocs #src_addr = addr_add(baseloc, ofs_loc, ofs.value, 0) assert ofs.value == 0 @@ -51,7 +36,7 @@ elif itemsize == 8: self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value) - def emit_vec_raw_load_i(self, op, arglocs, regalloc): + def emit_vec_load_i(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, \ Vhiloc, Vloloc, Vploc, tloc = arglocs #src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0) @@ -84,15 +69,15 @@ genop_discard_vec_setarrayitem_raw = _emit_vec_setitem genop_discard_vec_setarrayitem_gc = _emit_vec_setitem - def emit_vec_raw_store(self, op, arglocs, regalloc): + def emit_vec_store(self, op, arglocs, regalloc): baseloc, ofsloc, valueloc, size_loc, baseofs, \ integer_loc, aligned_loc = arglocs #dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0) assert baseofs.value == 0 - self._vec_store(baseloc, ofsloc, valueloc, integer_loc.value, - size_loc.value, regalloc) + # self._vec_store(baseloc, ofsloc, valueloc, integer_loc.value, + # size_loc.value, regalloc) - def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, regalloc): + #def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, regalloc): if integer: Vloloc = regalloc.ivrm.get_scratch_reg() Vhiloc = regalloc.ivrm.get_scratch_reg() @@ -137,7 +122,6 @@ elif itemsize == 8: self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value) - def emit_vec_int_add(self, op, arglocs, regalloc): resloc, loc0, loc1, size_loc = arglocs size = size_loc.value @@ -601,12 +585,8 @@ return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs), Vhiloc, Vloloc, Vploc, tloc] - prepare_vec_getarrayitem_raw_i = _prepare_load_i - prepare_vec_getarrayitem_raw_f = _prepare_load - prepare_vec_getarrayitem_gc_i = _prepare_load_i - prepare_vec_getarrayitem_gc_f = _prepare_load - prepare_vec_raw_load_i = _prepare_load_i - prepare_vec_raw_load_f = _prepare_load + prepare_vec_load_i = _prepare_load_i + prepare_vec_load_f = _prepare_load def prepare_vec_arith(self, op): a0 = op.getarg(0) @@ -639,7 +619,7 @@ del prepare_vec_arith - def _prepare_vec_store(self, op): + def prepare_vec_store(self, op): descr = op.getdescr() assert isinstance(descr, ArrayDescr) assert not descr.is_array_of_pointers() and \ @@ -657,11 +637,6 @@ return [baseloc, ofsloc, valueloc, imm(itemsize), imm(ofs), imm(integer), imm(aligned)] - prepare_vec_setarrayitem_raw = _prepare_vec_store - prepare_vec_setarrayitem_gc = _prepare_vec_store - prepare_vec_raw_store = _prepare_vec_store - del _prepare_vec_store - def prepare_vec_int_signext(self, op): assert isinstance(op, VectorOp) a0 = op.getarg(0) From pypy.commits at gmail.com Tue Jun 28 08:17:57 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 28 Jun 2016 05:17:57 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: resolve several issues to run test_vector again Message-ID: <57726af5.4e4ec20a.23eb8.48d0@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85425:647de5c9a91f Date: 2016-06-27 16:17 +0200 http://bitbucket.org/pypy/pypy/changeset/647de5c9a91f/ Log: resolve several issues to run test_vector again diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -70,15 +70,11 @@ genop_discard_vec_setarrayitem_gc = _emit_vec_setitem def emit_vec_store(self, op, arglocs, regalloc): - baseloc, ofsloc, valueloc, size_loc, baseofs, \ + baseloc, indexloc, valueloc, sizeloc, baseofs, \ integer_loc, aligned_loc = arglocs #dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0) assert baseofs.value == 0 - # self._vec_store(baseloc, ofsloc, valueloc, integer_loc.value, - # size_loc.value, regalloc) - - #def _vec_store(self, baseloc, indexloc, valueloc, integer, itemsize, regalloc): - if integer: + if integer_loc.value: Vloloc = regalloc.ivrm.get_scratch_reg() Vhiloc = regalloc.ivrm.get_scratch_reg() Vploc = regalloc.ivrm.get_scratch_reg() @@ -117,6 +113,7 @@ self.mc.stvx(Vlo, indexloc.value, t) self.mc.stvx(Vhi, indexloc.value, baseloc.value) else: + itemsize = sizeloc.value if itemsize == 4: self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value) elif itemsize == 8: diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -161,7 +161,6 @@ self.meta_interp(f, [l*size, va, vb, vc]) for i in range(l): - import pdb; pdb.set_trace() c = raw_storage_getitem(type,vc,i*size) r = func(la[i], lb[i]) assert isclose(r, c) From pypy.commits at gmail.com Tue Jun 28 08:17:59 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 28 Jun 2016 05:17:59 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: add more vector resop implementations (comparison operators) Message-ID: <57726af7.c3a9c20a.fdd59.5db2@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85426:973d081f4a33 Date: 2016-06-28 14:17 +0200 http://bitbucket.org/pypy/pypy/changeset/973d081f4a33/ Log: add more vector resop implementations (comparison operators) diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -67,6 +67,7 @@ XX3_2 = Form("fvrT", "fvrA", "fvrB", "OE", "XO11") XV = Form("ivrT", "rA", "rB", "XO1") VX = Form("ivrT", "ivrA", "ivrB", "XO8") +VC = Form("ivrT", "ivrA", "ivrB", "XO12", "OE") VXI = Form("ivrT", "SIM", "XO8") VA = Form("ivrT", "ivrA", "ivrB", "ivrC", "XO10") @@ -629,6 +630,29 @@ xvabsdp = XX2(60, XO6=473) xvabssp = XX2(60, XO6=409) + # conversion from/to + xvcvsxddp = XX2(60, XO6=504) + + # compare greater than unsigned int + vcmpgtubx = VC(4, XO12=518, OE=1) + vcmpgtub = VC(4, XO12=518, OE=0) + vcmpgtuhx = VC(4, XO12=584, OE=1) + vcmpgtuh = VC(4, XO12=584, OE=0) + vcmpgtuwx = VC(4, XO12=646, OE=1) + vcmpgtuw = VC(4, XO12=646, OE=0) + vcmpgtudx = VC(4, XO12=711, OE=1) + vcmpgtud = VC(4, XO12=711, OE=0) + + # compare equal to unsigned int + vcmpequbx = VC(4, XO12=6, OE=1) + vcmpequb = VC(4, XO12=6, OE=0) + vcmpequhx = VC(4, XO12=70, OE=1) + vcmpequh = VC(4, XO12=70, OE=0) + vcmpequwx = VC(4, XO12=134, OE=1) + vcmpequw = VC(4, XO12=134, OE=0) + vcmpequdx = VC(4, XO12=199, OE=1) + vcmpequd = VC(4, XO12=199, OE=0) + # INTEGER # ------- @@ -658,6 +682,7 @@ vand = VX(4, XO8=1028) vor = VX(4, XO8=1156) veqv = VX(4, XO8=1668) + vxor = VX(4, XO8=1220) # vector move register is alias to vector or vmr = vor diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py --- a/rpython/jit/backend/ppc/ppc_field.py +++ b/rpython/jit/backend/ppc/ppc_field.py @@ -65,6 +65,7 @@ "XO9": (21, 28), "XO10": (26, 31), "XO11": (22, 28), + "XO12": (22, 31), "LL": ( 9, 10), "SIM": (11, 15), } diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -13,6 +13,9 @@ from rpython.jit.backend.ppc.locations import imm from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN from rpython.jit.backend.llsupport.vector_ext import VectorExt +from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET +import rpython.jit.backend.ppc.register as r +import rpython.jit.backend.ppc.condition as c def not_implemented(msg): msg = '[ppc/vector_ext] %s\n' % msg @@ -146,7 +149,7 @@ elif size == 8: self.mc.vsubudm(resloc.value, loc0.value, loc1.value) - def emit_vec_float_add(self, op, arglocs, resloc): + def emit_vec_float_add(self, op, arglocs, regalloc): resloc, loc0, loc1, itemsize_loc = arglocs itemsize = itemsize_loc.value if itemsize == 4: @@ -154,7 +157,7 @@ elif itemsize == 8: self.mc.xvadddp(resloc.value, loc0.value, loc1.value) - def emit_vec_float_sub(self, op, arglocs, resloc): + def emit_vec_float_sub(self, op, arglocs, regalloc): resloc, loc0, loc1, itemsize_loc = arglocs itemsize = itemsize_loc.value if itemsize == 4: @@ -162,7 +165,7 @@ elif itemsize == 8: self.mc.xvsubdp(resloc.value, loc0.value, loc1.value) - def emit_vec_float_mul(self, op, arglocs, resloc): + def emit_vec_float_mul(self, op, arglocs, regalloc): resloc, loc0, loc1, itemsize_loc = arglocs itemsize = itemsize_loc.value if itemsize == 4: @@ -170,7 +173,7 @@ elif itemsize == 8: self.mc.xvmuldp(resloc.value, loc0.value, loc1.value) - def emit_vec_float_truediv(self, op, arglocs, resloc): + def emit_vec_float_truediv(self, op, arglocs, regalloc): resloc, loc0, loc1, itemsize_loc = arglocs itemsize = itemsize_loc.value if itemsize == 4: @@ -178,7 +181,8 @@ elif itemsize == 8: self.mc.xvdivdp(resloc.value, loc0.value, loc1.value) - def emit_vec_int_mul(self, op, arglocs, resloc): + def emit_vec_int_mul(self, op, arglocs, regalloc): + raise NotImplementedError pass # TODO def emit_vec_int_and(self, op, arglocs, regalloc): @@ -198,7 +202,7 @@ # TODO self.regalloc_mov(loc0, resloc) - def emit_vec_float_abs(self, op, arglocs, resloc): + def emit_vec_float_abs(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs size = sizeloc.value if size == 4: @@ -208,7 +212,7 @@ else: notimplemented("[ppc/assembler] float abs for size %d" % size) - def emit_vec_float_neg(self, op, arglocs, resloc): + def emit_vec_float_neg(self, op, arglocs, regalloc): resloc, argloc, sizeloc = arglocs size = sizeloc.value if size == 4: @@ -218,44 +222,45 @@ else: notimplemented("[ppc/assembler] float neg for size %d" % size) - #def genop_guard_vec_guard_true(self, guard_op, guard_token, locs, resloc): - # self.implement_guard(guard_token) + def emit_guard_vec_guard_true(self, guard_op, guard_token, arglocs, regalloc): + self._emit_guard(guard_op, arglocs) - #def genop_guard_vec_guard_false(self, guard_op, guard_token, locs, resloc): - # self.guard_success_cc = rx86.invert_condition(self.guard_success_cc) - # self.implement_guard(guard_token) + def emit_guard_vec_guard_false(self, guard_op, guard_token, arglocs, regalloc): + self.guard_success_cc = c.negate(self.guard_success_cc) + self._emit_guard(guard_op, arglocs) - #def guard_vector(self, guard_op, loc, true): + #def guard_vector(self, guard_op, regalloc, true): # assert isinstance(guard_op, VectorGuardOp) # arg = guard_op.getarg(0) # assert isinstance(arg, VectorOp) # size = arg.bytesize - # temp = X86_64_XMM_SCRATCH_REG + # temp = regalloc.get_scratch_reg().value # load = arg.bytesize * arg.count - self.cpu.vector_register_size - # assert load <= 0 + # assert load == 0 # if true: - # self.mc.PXOR(temp, temp) + # pass + # #self.mc.PXOR(temp, temp) # # if the vector is not fully packed blend 1s - # if load < 0: - # self.mc.PCMPEQQ(temp, temp) # fill with ones - # self._blend_unused_slots(loc, arg, temp) - # # reset to zeros - # self.mc.PXOR(temp, temp) + # #if load < 0: + # # self.mc.PCMPEQQ(temp, temp) # fill with ones + # # self._blend_unused_slots(loc, arg, temp) + # # # reset to zeros + # # self.mc.PXOR(temp, temp) # # cmp with zeros (in temp) creates ones at each slot where it is zero - # self.mc.PCMPEQ(loc, temp, size) - # # temp converted to ones - # self.mc.PCMPEQQ(temp, temp) - # # test if all slots are zero - # self.mc.PTEST(loc, temp) - # self.guard_success_cc = rx86.Conditions['Z'] + # #self.mc.PCMPEQ(loc, temp, size) + # ## temp converted to ones + # #self.mc.PCMPEQQ(temp, temp) + # ## test if all slots are zero + # #self.mc.PTEST(loc, temp) + # #self.guard_success_cc = rx86.Conditions['Z'] # else: # # if the vector is not fully packed blend 1s - # if load < 0: - # temp = X86_64_XMM_SCRATCH_REG - # self.mc.PXOR(temp, temp) - # self._blend_unused_slots(loc, arg, temp) - # self.mc.PTEST(loc, loc) + # #if load < 0: + # # temp = X86_64_XMM_SCRATCH_REG + # # self.mc.PXOR(temp, temp) + # # self._blend_unused_slots(loc, arg, temp) + # #self.mc.PTEST(loc, loc) # self.guard_success_cc = rx86.Conditions['NZ'] #def _blend_unused_slots(self, loc, arg, temp): @@ -322,51 +327,106 @@ # not_implemented("reduce sum for %s not impl." % arg) - #def genop_vec_int_is_true(self, op, arglocs, resloc): - # loc, sizeloc = arglocs - # temp = X86_64_XMM_SCRATCH_REG - # self.mc.PXOR(temp, temp) - # # every entry that is non zero -> becomes zero - # # zero entries become ones - # self.mc.PCMPEQ(loc, temp, sizeloc.value) - # # a second time -> every zero entry (corresponding to non zero - # # entries before) become ones - # self.mc.PCMPEQ(loc, temp, sizeloc.value) + def emit_vec_int_is_true(self, op, arglocs, regalloc): + resloc, argloc, sizeloc = arglocs + size = sizeloc.value + tmp = regalloc.get_scratch_reg().value + self.mc.vxor(tmp, tmp, tmp) + # argloc[i] > 0: + # For an unsigned integer that is equivalent to argloc[i] != 0 + if size == 1: + self.mc.vcmpgtubx(resloc.value, argloc.value, tmp) + elif size == 2: + self.mc.vcmpgtuhx(resloc.value, argloc.value, tmp) + elif size == 4: + self.mc.vcmpgtuwx(resloc.value, argloc.value, tmp) + elif size == 8: + self.mc.vcmpgtudx(resloc.value, argloc.value, tmp) - def emit_vec_float_eq(self, op, arglocs, resloc): + def emit_vec_float_eq(self, op, arglocs, regalloc): resloc, loc1, loc2, sizeloc = arglocs size = sizeloc.value + tmp = regalloc.vrm.get_scratch_reg().value + offloc = regalloc.rm.get_scratch_reg() + off = offloc.value + # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0 + self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) if size == 4: - self.mc.xvcmpeqspx(resloc.value, loc1.value, loc2.value) + self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value) + self.mc.stxvw4x(tmp, off, r.SP.value) elif size == 8: - self.mc.xvcmpeqdpx(resloc.value, loc1.value, loc2.value) + self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value) + self.mc.stxvd2x(tmp, off, r.SP.value) else: notimplemented("[ppc/assembler] float == for size %d" % size) + self.mc.lvx(resloc.value, off, r.SP.value) - def emit_vec_float_ne(self, op, arglocs, resloc): - self.emit_vec_float_eq(op, arglocs, resloc) + def emit_vec_float_ne(self, op, arglocs, regalloc): resloc, loc1, loc2, sizeloc = arglocs - self.mc.xxlandc(resloc.value, resloc.value, resloc.value) + size = sizeloc.value + tmp = regalloc.vrm.get_scratch_reg().value + offloc = regalloc.rm.get_scratch_reg() + off = offloc.value + # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0 + self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) + if size == 4: + self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value) + self.mc.xxlandc(tmp, tmp, tmp) # negate + self.mc.stxvw4x(tmp, off, r.SP.value) + elif size == 8: + self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value) + self.mc.xxlandc(tmp, tmp, tmp) # negate + self.mc.stxvd2x(tmp, off, r.SP.value) + else: + notimplemented("[ppc/assembler] float == for size %d" % size) + self.mc.lvx(resloc.value, off, r.SP.value) - #def genop_vec_int_eq(self, op, arglocs, resloc): - # _, rhsloc, sizeloc = arglocs - # size = sizeloc.value - # self.mc.PCMPEQ(resloc, rhsloc, size) + def emit_vec_cast_int_to_float(self, op, arglocs, regalloc): + res, l0 = arglocs + offloc = regalloc.rm.get_scratch_reg() + off = offloc.value + # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0 + self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) + self.mc.stvx(l0.value, off, r.SP.value) + self.mc.lxvd2x(res.value, off, r.SP.value) + self.mc.xvcvsxddp(res.value, res.value) - #def genop_vec_int_ne(self, op, arglocs, resloc): - # _, rhsloc, sizeloc = arglocs - # size = sizeloc.value - # self.mc.PCMPEQ(resloc, rhsloc, size) - # temp = X86_64_XMM_SCRATCH_REG - # self.mc.PCMPEQQ(temp, temp) # set all bits to one - # # need to invert the value in resloc - # self.mc.PXOR(resloc, temp) - # # 11 00 11 11 - # # 11 11 11 11 - # # ----------- pxor - # # 00 11 00 00 + def emit_vec_int_eq(self, op, arglocs, regalloc): + res, l0, l1, sizeloc = arglocs + size = sizeloc.value + if size == 1: + self.vcmpequbx(res.value, l0.value, l1.value) + elif size == 2: + self.vcmpequhx(res.value, l0.value, l1.value) + elif size == 4: + self.vcmpequwx(res.value, l0.value, l1.value) + elif size == 8: + self.vcmpequdx(res.value, l0.value, l1.value) - #def genop_vec_expand_f(self, op, arglocs, resloc): + def emit_vec_int_ne(self, op, arglocs, regalloc): + res, l0, l1, sizeloc = arglocs + size = sizeloc.value + tmp = regalloc.get_scratch_reg().value + self.mc.vxor(tmp, tmp, tmp) + if size == 1: + self.vcmpequb(res.value, l0.value, l1.value) + self.vcmpequbx(res.value, res.value, tmp) + elif size == 2: + self.vcmpequh(res.value, l0.value, l1.value) + self.vcmpequhx(res.value, res.value, tmp) + elif size == 4: + self.vcmpequw(res.value, l0.value, l1.value) + self.vcmpequwx(res.value, res.value, tmp) + elif size == 8: + self.vcmpequd(res.value, l0.value, l1.value) + self.vcmpequdx(res.value, res.value, tmp) + + #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): + # self.mc.CVTPD2DQ(resloc, arglocs[0]) + #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): + # self.mc.CVTPS2PD(resloc, arglocs[0]) + + #def genop_vec_expand_f(self, op, arglocs, regalloc): # srcloc, sizeloc = arglocs # size = sizeloc.value # if isinstance(srcloc, ConstFloatLoc): @@ -382,7 +442,7 @@ # else: # raise AssertionError("float of size %d not supported" % (size,)) - #def genop_vec_expand_i(self, op, arglocs, resloc): + #def genop_vec_expand_i(self, op, arglocs, regalloc): # srcloc, sizeloc = arglocs # if not isinstance(srcloc, RegLoc): # self.mov(srcloc, X86_64_SCRATCH_REG) @@ -406,7 +466,7 @@ # else: # raise AssertionError("cannot handle size %d (int expand)" % (size,)) - #def genop_vec_pack_i(self, op, arglocs, resloc): + #def genop_vec_pack_i(self, op, arglocs, regalloc): # resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs # assert isinstance(resultloc, RegLoc) # assert isinstance(sourceloc, RegLoc) @@ -516,18 +576,6 @@ #genop_vec_unpack_f = genop_vec_pack_f - #def genop_vec_cast_float_to_singlefloat(self, op, arglocs, resloc): - # self.mc.CVTPD2PS(resloc, arglocs[0]) - - #def genop_vec_cast_float_to_int(self, op, arglocs, resloc): - # self.mc.CVTPD2DQ(resloc, arglocs[0]) - - #def genop_vec_cast_int_to_float(self, op, arglocs, resloc): - # self.mc.CVTDQ2PD(resloc, arglocs[0]) - - #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, resloc): - # self.mc.CVTPS2PD(resloc, arglocs[0]) - class VectorRegalloc(object): _mixin_ = True @@ -718,13 +766,12 @@ # resloc = self.xrm.force_allocate_reg(op, args) # self.perform(op, [srcloc, imm(op.bytesize)], resloc) - #def prepare_vec_int_is_true(self, op): - # args = op.getarglist() - # arg = op.getarg(0) - # assert isinstance(arg, VectorOp) - # argloc = self.loc(arg) - # resloc = self.xrm.force_result_in_reg(op, arg, args) - # self.perform(op, [resloc,imm(arg.bytesize)], None) + def prepare_vec_int_is_true(self, op): + arg = op.getarg(0) + assert isinstance(arg, VectorOp) + argloc = self.ensure_vector_reg(arg) + resloc = self.force_allocate_vector_reg(op) + return [resloc, argloc, imm(arg.bytesize)] #def _prepare_vec(self, op): # # pseudo instruction, needed to create a new variable @@ -733,13 +780,12 @@ #prepare_vec_i = _prepare_vec #prepare_vec_f = _prepare_vec - #def prepare_vec_cast_float_to_int(self, op): - # args = op.getarglist() - # srcloc = self.make_sure_var_in_reg(op.getarg(0), args) - # resloc = self.xrm.force_result_in_reg(op, op.getarg(0), args) - # self.perform(op, [srcloc], resloc) + def prepare_vec_cast_float_to_int(self, op): + l0 = self.ensure_vector_reg(op.getarg(0)) + res = self.force_allocate_vector_reg(op) + return [res, l0] - #prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int + prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int #prepare_vec_cast_float_to_singlefloat = prepare_vec_cast_float_to_int #prepare_vec_cast_singlefloat_to_float = prepare_vec_cast_float_to_int diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -162,7 +162,7 @@ for i in range(l): c = raw_storage_getitem(type,vc,i*size) - r = func(la[i], lb[i]) + r = rffi.cast(type, func(la[i], lb[i])) assert isclose(r, c) rawstorage.clear() @@ -174,15 +174,18 @@ vec_float_binary = functools.partial(_vec_float_binary, _vector_simple_float) - test_vector_float_add = \ + test_vec_float_add = \ vec_float_binary(lambda a,b: a+b, rffi.DOUBLE) - test_vector_float_sub = \ + test_vec_float_sub = \ vec_float_binary(lambda a,b: a-b, rffi.DOUBLE) - test_vector_float_mul = \ + test_vec_float_mul = \ vec_float_binary(lambda a,b: a*b, rffi.DOUBLE) - #test_vector_float_div = \ + #test_vec_float_div = \ # vec_float_binary(lambda a,b: a/b, rffi.DOUBLE) + test_vec_float_cmp_eq = \ + vec_float_binary(lambda a,b: a == b, rffi.DOUBLE) + def _vector_simple_int(self, func, type, data): func = always_inline(func) @@ -225,76 +228,43 @@ vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int) - test_vector_signed_add = \ + test_vec_signed_add = \ vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED) - test_vector_int_add = \ + test_vec_int_add = \ vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT) - test_vector_short_add = \ + test_vec_short_add = \ vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT) - test_vector_signed_sub = \ + test_vec_signed_sub = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SIGNED) - test_vector_int_sub = \ + test_vec_int_sub = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT) - test_vector_short_sub = \ + test_vec_short_sub = \ vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT) - test_vector_signed_and = \ + test_vec_signed_and = \ vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SIGNED) - test_vector_int_and = \ + test_vec_int_and = \ vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.INT) - test_vector_short_and = \ + test_vec_short_and = \ vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SHORT) - test_vector_or_signed = \ + test_vec_or_signed = \ vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SIGNED) - test_vector_or_int = \ + test_vec_or_int = \ vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.INT) - test_vector_or_short = \ + test_vec_or_short = \ vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SHORT) - test_vector_xor_signed = \ + test_vec_xor_signed = \ vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SIGNED) - test_vector_xor_int = \ + test_vec_xor_int = \ vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.INT) - test_vector_xor_short = \ + test_vec_xor_short = \ vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT) - @py.test.mark.parametrize('i',[1,2,3,8,17,128,130,131,142,143]) - def test_vectorize_array_get_set(self,i): - myjitdriver = JitDriver(greens = [], - reds = 'auto', - vectorize=True) - T = lltype.Array(rffi.INT, hints={'nolength': True}) - def f(d): - i = 0 - va = lltype.malloc(T, d, flavor='raw', zero=True) - vb = lltype.malloc(T, d, flavor='raw', zero=True) - vc = lltype.malloc(T, d, flavor='raw', zero=True) - for j in range(d): - va[j] = rffi.r_int(j) - vb[j] = rffi.r_int(j) - while i < d: - myjitdriver.jit_merge_point() - - a = va[i] - b = vb[i] - ec = intmask(a)+intmask(b) - vc[i] = rffi.r_int(ec) - - i += 1 - res = 0 - for j in range(d): - res += intmask(vc[j]) - lltype.free(va, flavor='raw') - lltype.free(vb, flavor='raw') - lltype.free(vc, flavor='raw') - return res - res = self.meta_interp(f, [i]) - assert res == f(i) - @py.test.mark.parametrize('i',[1,2,3,4,9]) - def test_vector_register_too_small_vector(self, i): + def test_vec_register_too_small_vector(self, i): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) @@ -328,7 +298,7 @@ res = self.meta_interp(f, [i]) assert res == f(i) == 3 - def test_vectorize_max(self): + def test_vec_max(self): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) From pypy.commits at gmail.com Tue Jun 28 08:47:21 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:21 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: merge default Message-ID: <577271d9.0f941c0a.c33ee.0f32@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85428:5641802cb054 Date: 2016-06-24 17:33 +0200 http://bitbucket.org/pypy/pypy/changeset/5641802cb054/ Log: merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,6 +1,6 @@ -========================= +========================== What's new in PyPy2.7 5.3+ -========================= +========================== .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 @@ -37,3 +37,14 @@ .. branch: pyfile-tell Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile + +.. branch: rw-PyString_AS_STRING +Allow rw access to the char* returned from PyString_AS_STRING, also refactor +PyStringObject to look like cpython's and allow subclassing PyString_Type and +PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -800,6 +800,21 @@ pypy_debug_catch_fatal_exception() assert False +def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid): + from rpython.rlib import rgil + # see "Handling of the GIL" above + assert cpyext_glob_tid_ptr[0] == 0 + if pygilstate_release: + from pypy.module.cpyext import pystate + unlock = (gilstate == pystate.PyGILState_UNLOCKED) + else: + unlock = gil_release or _gil_auto + if unlock: + rgil.release() + else: + cpyext_glob_tid_ptr[0] = tid + + def make_wrapper_second_level(space, argtypesw, restype, result_kind, error_value, gil): from rpython.rlib import rgil @@ -827,6 +842,7 @@ def wrapper_second_level(callable, pname, *args): from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj from pypy.module.cpyext.pyobject import as_pyobj + from pypy.module.cpyext import pystate # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -839,7 +855,6 @@ rgil.acquire() assert cpyext_glob_tid_ptr[0] == 0 elif pygilstate_ensure: - from pypy.module.cpyext import pystate if cpyext_glob_tid_ptr[0] == tid: cpyext_glob_tid_ptr[0] = 0 args += (pystate.PyGILState_LOCKED,) @@ -850,6 +865,10 @@ if cpyext_glob_tid_ptr[0] != tid: no_gil_error(pname) cpyext_glob_tid_ptr[0] = 0 + if pygilstate_release: + gilstate = rffi.cast(lltype.Signed, args[-1]) + else: + gilstate = pystate.PyGILState_IGNORE rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -919,24 +938,13 @@ except Exception as e: unexpected_exception(pname, e, tb) + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return fatal_value assert lltype.typeOf(retval) == restype rffi.stackcounter.stacks_counter -= 1 - # see "Handling of the GIL" above - assert cpyext_glob_tid_ptr[0] == 0 - if pygilstate_release: - from pypy.module.cpyext import pystate - arg = rffi.cast(lltype.Signed, args[-1]) - unlock = (arg == pystate.PyGILState_UNLOCKED) - else: - unlock = gil_release or _gil_auto - if unlock: - rgil.release() - else: - cpyext_glob_tid_ptr[0] = tid - + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return retval wrapper_second_level._dont_inline_ = True @@ -1202,8 +1210,6 @@ cpyext_type_init = self.cpyext_type_init self.cpyext_type_init = None for pto, w_type in cpyext_type_init: - if space.is_w(w_type, space.w_str): - pto.c_tp_itemsize = 1 finish_type_1(space, pto) finish_type_2(space, pto, w_type) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -6,7 +6,9 @@ from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, - make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref) + make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref, + pyobj_has_w_obj) +from pypy.objspace.std.bytesobject import W_BytesObject ## ## Implementation of PyStringObject @@ -16,7 +18,7 @@ ## ----------- ## ## PyString_AsString() must return a (non-movable) pointer to the underlying -## buffer, whereas pypy strings are movable. C code may temporarily store +## ob_sval, whereas pypy strings are movable. C code may temporarily store ## this address and use it, as long as it owns a reference to the PyObject. ## There is no "release" function to specify that the pointer is not needed ## any more. @@ -28,7 +30,7 @@ ## -------- ## ## PyStringObject contains two additional members: the ob_size and a pointer to a -## char buffer; it may be NULL. +## char ob_sval; it may be NULL. ## ## - A string allocated by pypy will be converted into a PyStringObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is @@ -41,6 +43,9 @@ ## the pypy string is created, and added to the global map of tracked ## objects. The buffer is then supposed to be immutable. ## +##- A buffer obtained from PyString_AS_STRING() could be mutable iff +## there is no corresponding pypy object for the string +## ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a ## similar object. ## @@ -53,7 +58,7 @@ PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) PyStringObjectFields = PyVarObjectFields + \ - (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP)) + (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) @bootstrap_function @@ -69,44 +74,43 @@ def new_empty_str(space, length): """ - Allocate a PyStringObject and its buffer, but without a corresponding - interpreter object. The buffer may be mutated, until string_realize() is + Allocate a PyStringObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until string_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) - py_obj = typedescr.allocate(space, space.w_str) + py_obj = typedescr.allocate(space, space.w_str, length) py_str = rffi.cast(PyStringObject, py_obj) - - buflen = length + 1 - py_str.c_ob_size = length - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, - flavor='raw', zero=True, - add_memory_pressure=True) + py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str def string_attach(space, py_obj, w_obj): """ - Fills a newly allocated PyStringObject with the given string object. The - buffer must not be modified. + Copy RPython string object contents to a PyStringObject. The + c_ob_sval must not be modified. """ py_str = rffi.cast(PyStringObject, py_obj) - py_str.c_ob_size = len(space.str_w(w_obj)) - py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + s = space.str_w(w_obj) + if py_str.c_ob_size < len(s): + raise oefmt(space.w_ValueError, + "string_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) + rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) + py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL def string_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject buffer must not + Creates the string in the interpreter. The PyStringObject ob_sval must not be modified after this call. """ py_str = rffi.cast(PyStringObject, py_obj) - if not py_str.c_buffer: - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1, - flavor='raw', zero=True) - s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size) - w_obj = space.wrap(s) + s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_BytesObject, w_type) + w_obj.__init__(s) py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL track_reference(space, py_obj, w_obj) @@ -116,9 +120,6 @@ def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ - py_str = rffi.cast(PyStringObject, py_obj) - if py_str.c_buffer: - lltype.free(py_str.c_buffer, flavor="raw") from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) @@ -139,6 +140,9 @@ @cpython_api([PyObject], rffi.CCHARP, error=0) def PyString_AsString(space, ref): + return _PyString_AsString(space, ref) + +def _PyString_AsString(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: pass # typecheck returned "ok" without forcing 'ref' at all elif not PyString_Check(space, ref): # otherwise, use the alternate way @@ -151,15 +155,23 @@ "expected string or Unicode object, %T found", from_ref(space, ref)) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - return ref_str.c_buffer + if not pyobj_has_w_obj(ref): + # XXX Force the ref? + string_realize(space, ref) + return ref_str.c_ob_sval + + at cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) +def PyString_AS_STRING(space, void_ref): + ref = rffi.cast(PyObject, void_ref) + # if no w_str is associated with this ref, + # return the c-level ptr as RW + if not pyobj_has_w_obj(ref): + py_str = rffi.cast(PyStringObject, ref) + return py_str.c_ob_sval + return _PyString_AsString(space, ref) @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) -def PyString_AsStringAndSize(space, ref, buffer, length): +def PyString_AsStringAndSize(space, ref, data, length): if not PyString_Check(space, ref): from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) @@ -169,18 +181,16 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) + if not pyobj_has_w_obj(ref): + # force the ref + string_realize(space, ref) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - buffer[0] = ref_str.c_buffer + data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size else: i = 0 - while ref_str.c_buffer[i] != '\0': + while ref_str.c_ob_sval[i] != '\0': i += 1 if i != ref_str.c_ob_size: raise oefmt(space.w_TypeError, @@ -209,10 +219,10 @@ set to NULL, a memory exception is set, and -1 is returned. """ # XXX always create a new string so far - py_str = rffi.cast(PyStringObject, ref[0]) - if not py_str.c_buffer: + if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") + py_str = rffi.cast(PyStringObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: @@ -224,7 +234,7 @@ if oldsize < newsize: to_cp = oldsize for i in range(to_cp): - py_newstr.c_buffer[i] = py_str.c_buffer[i] + py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i] Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 diff --git a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h --- a/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/__multiarray_api.h @@ -5,7 +5,12 @@ npy_bool obval; } PyBoolScalarObject; -static int import_array(){return 0;}; -static int _import_array(){return 0;}; -static int _import_math(){return 0;}; +#if PY_VERSION_HEX >= 0x03000000 +#define NUMPY_IMPORT_ARRAY_RETVAL NULL +#else +#define NUMPY_IMPORT_ARRAY_RETVAL +#endif +#define import_array() {return NUMPY_IMPORT_ARRAY_RETVAL;} + + diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h --- a/pypy/module/cpyext/include/stringobject.h +++ b/pypy/module/cpyext/include/stringobject.h @@ -10,7 +10,6 @@ #include #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op)) -#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op)) /* Type PyStringObject represents a character string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is @@ -41,12 +40,11 @@ PyObject_VAR_HEAD long ob_shash; int ob_sstate; - char * buffer; /* change the name from cpython so all non-api c access is thwarted */ + char ob_sval[1]; /* Invariants - * (not relevant in PyPy, all stringobjects are backed by a pypy object) - * buffer contains space for 'ob_size+1' elements. - * buffer[ob_size] == 0. + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. * ob_shash is the hash of the string or -1 if not computed yet. * ob_sstate != 0 iff the string object is in stringobject.c's * 'interned' dictionary; in this case the two references diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, - INTERPLEVEL_API) + INTERPLEVEL_API, PyVarObject) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject @@ -47,13 +47,16 @@ size = pytype.c_tp_basicsize else: size = rffi.sizeof(self.basestruct) - if itemcount and w_type is not space.w_str: + if pytype.c_tp_itemsize: size += itemcount * pytype.c_tp_itemsize assert size >= rffi.sizeof(PyObject.TO) buf = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True, add_memory_pressure=True) pyobj = rffi.cast(PyObject, buf) + if pytype.c_tp_itemsize: + pyvarobj = rffi.cast(PyVarObject, pyobj) + pyvarobj.c_ob_size = itemcount pyobj.c_ob_refcnt = 1 #pyobj.c_ob_pypy_link should get assigned very quickly pyobj.c_ob_type = pytype @@ -152,13 +155,18 @@ class InvalidPointerException(Exception): pass -def create_ref(space, w_obj, itemcount=0): +def create_ref(space, w_obj): """ Allocates a PyObject, and fills its fields with info from the given interpreter object. """ w_type = space.type(w_obj) + pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) + if pytype.c_tp_itemsize != 0: + itemcount = space.len_w(w_obj) # PyStringObject and subclasses + else: + itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) track_reference(space, py_obj, w_obj) # diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -207,6 +207,7 @@ PyGILState_STATE = rffi.INT PyGILState_LOCKED = 0 PyGILState_UNLOCKED = 1 +PyGILState_IGNORE = 2 ExecutionContext.cpyext_gilstate_counter_noleave = 0 diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c --- a/pypy/module/cpyext/src/stringobject.c +++ b/pypy/module/cpyext/src/stringobject.c @@ -107,7 +107,7 @@ if (!string) return NULL; - s = PyString_AsString(string); + s = PyString_AS_STRING(string); for (f = format; *f; f++) { if (*f == '%') { diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,4 +1,4 @@ -import py +import os import pytest def pytest_configure(config): @@ -21,3 +21,14 @@ def pytest_funcarg__api(request): return request.cls.api +if os.name == 'nt': + @pytest.yield_fixture(autouse=True, scope='session') + def prevent_dialog_box(): + """Do not open dreaded dialog box on segfault on Windows""" + import ctypes + SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN + old_err_mode = ctypes.windll.kernel32.GetErrorMode() + new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX + ctypes.windll.kernel32.SetErrorMode(new_err_mode) + yield + ctypes.windll.kernel32.SetErrorMode(old_err_mode) diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/support.py @@ -0,0 +1,68 @@ +import os +import py +from sys import platform + +if os.name != 'nt': + so_ext = 'so' +else: + so_ext = 'dll' + +def c_compile(cfilenames, outputfilename, + compile_extra=None, link_extra=None, + include_dirs=None, libraries=None, library_dirs=None): + compile_extra = compile_extra or [] + link_extra = link_extra or [] + include_dirs = include_dirs or [] + libraries = libraries or [] + library_dirs = library_dirs or [] + if platform == 'win32': + link_extra = link_extra + ['/DEBUG'] # generate .pdb file + if platform == 'darwin': + # support Fink & Darwinports + for s in ('/sw/', '/opt/local/'): + if (s + 'include' not in include_dirs + and os.path.exists(s + 'include')): + include_dirs.append(s + 'include') + if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): + library_dirs.append(s + 'lib') + + outputfilename = py.path.local(outputfilename).new(ext=so_ext) + saved_environ = os.environ.copy() + try: + _build( + cfilenames, outputfilename, + compile_extra, link_extra, + include_dirs, libraries, library_dirs) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(cfilenames, outputfilename, compile_extra, link_extra, + include_dirs, libraries, library_dirs): + from distutils.ccompiler import new_compiler + from distutils import sysconfig + compiler = new_compiler(force=1) + sysconfig.customize_compiler(compiler) # XXX + objects = [] + for cfile in cfilenames: + cfile = py.path.local(cfile) + old = cfile.dirpath().chdir() + try: + res = compiler.compile([cfile.basename], + include_dirs=include_dirs, extra_preargs=compile_extra) + assert len(res) == 1 + cobjfile = py.path.local(res[0]) + assert cobjfile.check() + objects.append(str(cobjfile)) + finally: + old.chdir() + + compiler.link_shared_object( + objects, str(outputfilename), + libraries=libraries, + extra_preargs=link_extra, + library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestStringObject(AppTestCpythonExtensionBase): def test_basic(self): module = self.import_extension('foo', [ @@ -16,24 +17,10 @@ """ PyObject* s = PyByteArray_FromStringAndSize("Hello world", 12); int result = 0; - size_t expected_size; if(PyByteArray_Size(s) == 12) { result = 1; } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*3; - #elif defined Py_DEBUG - expected_size = 64; - #else - expected_size = 48; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); return PyBool_FromLong(result); """), @@ -53,7 +40,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyByteArray_FromStringAndSize(NULL, 4); if (s == NULL) @@ -84,11 +70,10 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); - memcpy(PyByteArray_AS_STRING(base), "works", 6); + memcpy(PyByteArray_AS_STRING(base), "works", 6); Py_INCREF(base); return base; """), @@ -115,6 +100,7 @@ assert s == 'test' def test_manipulations(self): + import sys module = self.import_extension('foo', [ ("bytearray_from_string", "METH_VARARGS", ''' @@ -141,9 +127,9 @@ ("concat", "METH_VARARGS", """ PyObject * ret, *right, *left; - PyObject *ba1, *ba2; + PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -157,7 +143,9 @@ """)]) assert module.bytearray_from_string("huheduwe") == "huhe" assert module.str_from_bytearray(bytearray('abc')) == 'abc' - raises(ValueError, module.str_from_bytearray, 4.0) + if '__pypy__' in sys.builtin_module_names: + # CPython only makes an assert. + raises(ValueError, module.str_from_bytearray, 4.0) ret = module.concat('abc', 'def') assert ret == 'abcdef' assert not isinstance(ret, str) @@ -171,9 +159,9 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } - + ba = PyByteArray_FromObject(obj); if (ba == NULL) return NULL; @@ -187,7 +175,7 @@ { printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); return NULL; - } + } return ba; ''' )]) diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,31 +25,15 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyString_FromString("Hello world"); - int result = 0; - size_t expected_size; + int result = PyString_Size(s); - if(PyString_Size(s) == 11) { - result = 1; - } - #ifdef PYPY_VERSION - expected_size = sizeof(void*)*7; - #elif defined Py_DEBUG - expected_size = 53; - #else - expected_size = 37; - #endif - if(s->ob_type->tp_basicsize != expected_size) - { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); - result = 0; - } Py_DECREF(s); - return PyBool_FromLong(result); + return PyLong_FromLong(result); """), ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyString_Size(f); + PyString_Size(f); Py_DECREF(f); return NULL; @@ -60,7 +44,7 @@ """)], prologue='#include ') assert module.get_hello1() == 'Hello world' assert module.get_hello2() == 'Hello world' - assert module.test_Size() + assert module.test_Size() == 11 raises(TypeError, module.test_Size_exception) assert module.test_is_string("") @@ -72,7 +56,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyString_FromStringAndSize(NULL, 4); if (s == NULL) @@ -81,7 +64,7 @@ if (t == NULL) return NULL; Py_DECREF(t); - c = PyString_AsString(s); + c = PyString_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -100,7 +83,6 @@ PyObject *base; PyTypeObject * type; PyStringObject *obj; - char * p_str; base = PyString_FromString("test"); if (PyString_GET_SIZE(base) != 4) return PyLong_FromLong(-PyString_GET_SIZE(base)); @@ -110,14 +92,22 @@ obj = (PyStringObject*)type->tp_alloc(type, 10); if (PyString_GET_SIZE(obj) != 10) return PyLong_FromLong(PyString_GET_SIZE(obj)); - /* cannot work, there is only RO access - memcpy(PyString_AS_STRING(obj), "works", 6); */ + /* cannot work, there is only RO access */ + memcpy(PyString_AS_STRING(obj), "works", 6); Py_INCREF(obj); return (PyObject*)obj; """), + ('alloc_rw', "METH_NOARGS", + ''' + PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); + memcpy(PyString_AS_STRING(obj), "works", 6); + return (PyObject*)obj; + '''), ]) + s = module.alloc_rw() + assert s == 'works' + '\x00' * 5 s = module.tpalloc() - assert s == '\x00' * 10 + assert s == 'works' + '\x00' * 5 def test_AsString(self): module = self.import_extension('foo', [ @@ -312,17 +302,17 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyStringObject*)obj)->ob_shash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - int sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*int sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); '''), @@ -332,27 +322,124 @@ # doesn't really test, but if printf is enabled will prove sstate assert module.test_sstate() + def test_subclass(self): + # taken from PyStringArrType_Type in numpy's scalartypes.c.src + module = self.import_extension('bar', [ + ("newsubstr", "METH_O", + """ + PyObject * obj; + char * data; + int len; + PyType_Ready(&PyStringArrType_Type); + + data = PyString_AS_STRING(args); + len = PyString_GET_SIZE(args); + if (data == NULL || len < 1) + Py_RETURN_NONE; + obj = PyArray_Scalar(data, len); + return obj; + """), + ], prologue=""" + #include + PyTypeObject PyStringArrType_Type = { + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ + "bar.string_", /* tp_name*/ + sizeof(PyStringObject), /* tp_basicsize*/ + 0 /* tp_itemsize */ + }; + + static PyObject * + stringtype_repr(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + static PyObject * + stringtype_str(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + PyObject * + PyArray_Scalar(char *data, int n) + { + PyTypeObject *type = &PyStringArrType_Type; + PyObject *obj; + void *destptr; + int itemsize = n; + obj = type->tp_alloc(type, itemsize); + if (obj == NULL) { + return NULL; + } + destptr = PyString_AS_STRING(obj); + ((PyStringObject *)obj)->ob_shash = -1; + memcpy(destptr, data, itemsize); + return obj; + } + """, more_init = ''' + PyStringArrType_Type.tp_alloc = NULL; + PyStringArrType_Type.tp_free = NULL; + + PyStringArrType_Type.tp_repr = stringtype_repr; + PyStringArrType_Type.tp_str = stringtype_str; + PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE; + PyStringArrType_Type.tp_itemsize = sizeof(char); + PyStringArrType_Type.tp_base = &PyString_Type; + ''') + + a = module.newsubstr('abc') + assert type(a).__name__ == 'string_' + assert a == 'abc' class TestString(BaseApiTest): def test_string_resize(self, space, api): py_str = new_empty_str(space, 10) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - py_str.c_buffer[0] = 'a' - py_str.c_buffer[1] = 'b' - py_str.c_buffer[2] = 'c' + py_str.c_ob_sval[0] = 'a' + py_str.c_ob_sval[1] = 'b' + py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 3) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 3 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[3] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 10) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 10 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[10] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[10] == '\x00' Py_DecRef(space, ar[0]) lltype.free(ar, flavor='raw') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -7,8 +7,6 @@ from pypy import pypydir from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator import platform from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir from pypy.module.cpyext import api @@ -18,20 +16,7 @@ from rpython.tool import leakfinder from rpython.rlib import rawrefcount -def setup_module(module): - if os.name == 'nt': - # Do not open dreaded dialog box on segfault - import ctypes - SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN - old_err_mode = ctypes.windll.kernel32.GetErrorMode() - new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX - ctypes.windll.kernel32.SetErrorMode(new_err_mode) - module.old_err_mode = old_err_mode - -def teardown_module(module): - if os.name == 'nt': - import ctypes - ctypes.windll.kernel32.SetErrorMode(module.old_err_mode) +from .support import c_compile @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -46,7 +31,30 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def compile_extension_module(space, modname, include_dirs=[], **kwds): +def convert_sources_to_files(sources, dirname): + files = [] + for i, source in enumerate(sources): + filename = dirname / ('source_%d.c' % i) + with filename.open('w') as f: + f.write(str(source)) + files.append(filename) + return files + +def create_so(modname, include_dirs, source_strings=None, source_files=None, + compile_extra=None, link_extra=None, libraries=None): + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=compile_extra, link_extra=link_extra, + include_dirs=include_dirs, + libraries=libraries) + return soname + +def compile_extension_module(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -60,38 +68,36 @@ state = space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': - kwds["libraries"] = [api_library] + libraries = [api_library] # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] + compile_extra = ["/we4013"] # prevent linking with PythonXX.lib w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] - kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % (space.int_w(w_maj), space.int_w(w_min))] - elif sys.platform == 'darwin': - kwds["link_files"] = [str(api_library + '.dylib')] else: - kwds["link_files"] = [str(api_library + '.so')] + libraries = [] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror", "-g", "-O0"] - kwds["link_extra"]=["-g"] + compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + link_extra = ["-g"] + else: + compile_extra = link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs=api.include_dirs + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=api.include_dirs + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra, + libraries=libraries) from pypy.module.imp.importing import get_so_extension pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) soname.rename(pydname) return str(pydname) -def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds): +def compile_extension_module_applevel(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -103,24 +109,23 @@ build the module (so specify your source with one of those). """ if sys.platform == 'win32': - kwds["compile_extra"] = ["/we4013"] - kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] + compile_extra = ["/we4013"] + link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] elif sys.platform == 'darwin': + compile_extra = link_extra = None pass elif sys.platform.startswith('linux'): - kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"] + compile_extra = [ + "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] + link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs = [space.include_dir] + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=[space.include_dir] + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra) return str(soname) def freeze_refcnts(self): @@ -285,8 +290,8 @@ separate_module_sources = [] pydname = self.compile_extension_module( space, name, - separate_module_files=separate_module_files, - separate_module_sources=separate_module_sources) + source_files=separate_module_files, + source_strings=separate_module_sources) return space.wrap(pydname) @gateway.unwrap_spec(name=str, init='str_or_None', body=str, @@ -315,6 +320,11 @@ /* fix for cpython 2.7 Python.h if running tests with -A since pypy compiles with -fvisibility-hidden */ #undef PyMODINIT_FUNC + #ifdef __GNUC__ + # define RPY_EXPORTED extern __attribute__((visibility("default"))) + #else + # define RPY_EXPORTED extern __declspec(dllexport) + #endif #define PyMODINIT_FUNC RPY_EXPORTED void %(body)s @@ -326,16 +336,16 @@ """ % dict(name=name, init=init, body=body, PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' if PY_SSIZE_T_CLEAN else '') - kwds = dict(separate_module_sources=[code]) + kwds = dict(source_strings=[code]) else: assert not PY_SSIZE_T_CLEAN if filename is None: filename = name filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(separate_module_files=[filename]) - kwds['include_dirs'] = include_dirs - mod = self.compile_extension_module(space, name, **kwds) + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module(space, name, + include_dirs=include_dirs, **kwds) if load_it: if self.runappdirect: @@ -975,7 +985,7 @@ ('bar', 'METH_NOARGS', ''' /* reuse a name that is #defined in structmember.h */ - int RO; + int RO = 0; (void)RO; Py_RETURN_NONE; ''' ), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -142,7 +142,7 @@ 2000, 6, 6, 6, 6, 6, 6, Py_None, PyDateTimeAPI->DateTimeType); """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.new_date() == datetime.date(2000, 6, 6) assert module.new_time() == datetime.time(6, 6, 6, 6) @@ -241,6 +241,9 @@ PyObject* obj = PyDelta_FromDSU(6, 6, 6); PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 + // These macros are only defined in CPython 3.x and PyPy. + // See: http://bugs.python.org/issue13727 PyDateTime_DELTA_GET_DAYS(obj); PyDateTime_DELTA_GET_DAYS(delta); @@ -249,10 +252,10 @@ PyDateTime_DELTA_GET_MICROSECONDS(obj); PyDateTime_DELTA_GET_MICROSECONDS(delta); - +#endif return obj; """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.test_date_macros() == datetime.date(2000, 6, 6) assert module.test_datetime_macros() == datetime.datetime( diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py --- a/pypy/module/cpyext/test/test_frameobject.py +++ b/pypy/module/cpyext/test/test_frameobject.py @@ -13,7 +13,7 @@ PyObject *empty_string = PyString_FromString(""); PyObject *empty_tuple = PyTuple_New(0); PyCodeObject *py_code; - PyFrameObject *py_frame; + PyFrameObject *py_frame = NULL; py_code = PyCode_New( 0, /*int argcount,*/ @@ -75,7 +75,7 @@ """ int check; PyObject *type, *value, *tb; - PyObject *ret = PyRun_String("XXX", Py_eval_input, + PyObject *ret = PyRun_String("XXX", Py_eval_input, Py_None, Py_None); if (ret) { Py_DECREF(ret); diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -149,7 +149,6 @@ pybuffer = self.import_parser( ''' Py_buffer buf1, buf2, buf3; - PyObject *result; if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) { return NULL; } diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.micronumpy.ndarray import W_NDimArray from pypy.module.micronumpy.descriptor import get_dtype_cache -import pypy.module.micronumpy.constants as NPY +import pypy.module.micronumpy.constants as NPY def scalar(space): dtype = get_dtype_cache(space).w_float64dtype @@ -237,7 +237,7 @@ except: skip('numpy not importable') else: - numpy_incl = os.path.abspath(os.path.dirname(__file__) + + numpy_incl = os.path.abspath(os.path.dirname(__file__) + '/../include/_numpypy') assert os.path.exists(numpy_incl) cls.w_numpy_include = cls.space.wrap([numpy_incl]) @@ -273,7 +273,7 @@ { /* Should have failed */ Py_DECREF(obj1); - return NULL; + return NULL; } return obj1; ''' @@ -300,14 +300,14 @@ ), ("test_DescrFromType", "METH_O", """ - Signed typenum = PyInt_AsLong(args); + long typenum = PyInt_AsLong(args); return PyArray_DescrFromType(typenum); """ ), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -315,7 +315,7 @@ #define PyArray_FromObject _PyArray_FromObject #define PyArray_FromAny _PyArray_FromAny #endif - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -349,14 +349,14 @@ Py_INCREF(obj); return obj; '''), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -403,14 +403,14 @@ void *array_data[] = {NULL, NULL}; return PyUFunc_FromFuncAndDataAndSignature(funcs, array_data, types, 1, 1, 1, PyUFunc_None, - "float_3x3", - "a ufunc that tests a more complicated signature", + "float_3x3", + "a ufunc that tests a more complicated signature", 0, "(m,m)->(m,m)"); """), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include + #include #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include @@ -480,7 +480,7 @@ res += +10; *((float *)args[1]) = res; }; - + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -127,12 +127,12 @@ test_compare(1, 2) test_compare(2, 2) test_compare('2', '1') - + w_i = space.wrap(1) assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() - + def test_IsInstance(self, space, api): assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 @@ -165,7 +165,7 @@ return File""") w_f = space.call_function(w_File) assert api.PyObject_AsFileDescriptor(w_f) == 42 - + def test_hash(self, space, api): assert api.PyObject_Hash(space.wrap(72)) == 72 assert api.PyObject_Hash(space.wrap(-1)) == -1 @@ -250,7 +250,7 @@ if (copy != orig) PyObject_Free(copy); PyObject_Free(orig); - return ret; + return ret; """)]) x = module.realloctest() assert x == 'hello world\x00' @@ -425,7 +425,6 @@ """ Py_buffer buf; PyObject *str = PyString_FromString("hello, world."); - PyObject *result; if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13, 1, PyBUF_WRITABLE)) { diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -130,7 +130,7 @@ module = self.import_extension('foo', [ ("run", "METH_NOARGS", """ - long prev, next; + long prev; PyObject *t = PyTuple_New(1); prev = Py_True->ob_refcnt; Py_INCREF(Py_True); diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,8 +5,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr -import sys - class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): import sys @@ -737,7 +735,6 @@ """ IntLikeObject *intObj; int intval; - PyObject *name; if (!PyArg_ParseTuple(args, "i", &intval)) return NULL; @@ -897,7 +894,7 @@ module.footype("X", (object,), {}) def test_app_subclass_of_c_type(self): - # on cpython, the size changes (6 bytes added) + import sys module = self.import_module(name='foo') size = module.size_of_instances(module.fooType) class f1(object): @@ -907,7 +904,11 @@ class bar(f1, f2): pass assert bar.__base__ is f2 - assert module.size_of_instances(bar) == size + # On cpython, the size changes. + if '__pypy__' in sys.builtin_module_names: + assert module.size_of_instances(bar) == size + else: + assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): module = self.import_module(name='foo') @@ -1058,7 +1059,6 @@ module = self.import_extension('foo', [ ("getMetaClass", "METH_NOARGS", ''' - PyObject *obj; FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT; FooType_Type.tp_base = &PyType_Type; if (PyType_Ready(&FooType_Type) < 0) return NULL; diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -35,7 +35,7 @@ ("test_GetSize_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyUnicode_GetSize(f); + PyUnicode_GetSize(f); Py_DECREF(f); return NULL; @@ -57,7 +57,6 @@ """ PyObject *s, *t; Py_UNICODE* c; - Py_ssize_t len; s = PyUnicode_FromUnicode(NULL, 4); if (s == NULL) @@ -84,7 +83,7 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyUnicodeObject*)obj)->hash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ]) @@ -234,13 +233,13 @@ w_res = api.PyUnicode_AsUTF8String(w_u) assert space.type(w_res) is space.w_str assert space.unwrap(w_res) == 'sp\tm' - + def test_decode_utf8(self, space, api): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) w_u = api.PyUnicode_DecodeUTF8(u, 5, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == u'sp\x134m' - + w_u = api.PyUnicode_DecodeUTF8(u, 2, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == 'sp' @@ -405,7 +404,7 @@ ustr = "abcdef" w_ustr = space.wrap(ustr.decode("ascii")) result = api.PyUnicode_AsASCIIString(w_ustr) - + assert space.eq_w(space.wrap(ustr), result) w_ustr = space.wrap(u"abcd\xe9f") diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -657,6 +657,8 @@ pto.c_tp_dealloc = llhelper( subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) + if space.is_w(w_type, space.w_str): + pto.c_tp_itemsize = 1 # buffer protocol setup_buffer_procs(space, w_type, pto) @@ -695,6 +697,8 @@ if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize + if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize: + pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize # will be filled later on with the correct value # may not be 0 diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -77,7 +77,9 @@ """ py_uni = rffi.cast(PyUnicodeObject, py_obj) s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length) - w_obj = space.wrap(s) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type) + w_obj.__init__(s) py_uni.c_hash = space.hash_w(w_obj) track_reference(space, py_obj, w_obj) return w_obj diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -481,7 +481,7 @@ self.intbound = info def _generate_guards_unkown(self, other, box, runtime_box, extra_guards, - optimizer): + state): other_intbound = None if isinstance(other, NotVirtualStateInfoInt): other_intbound = other.intbound @@ -493,7 +493,7 @@ self.intbound.contains(runtime_box.getint())): # this may generate a few more guards than needed, but they are # optimized away when emitting them - self.intbound.make_guards(box, extra_guards, optimizer) + self.intbound.make_guards(box, extra_guards, state.optimizer) return raise VirtualStatesCantMatch("intbounds don't match") diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -487,7 +487,9 @@ #hstrerror.argtypes = [c_int] #hstrerror.restype = c_char_p -socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type) +socket = external('socket', [rffi.INT, rffi.INT, rffi.INT], socketfd_type, + save_err=SAVE_ERR) + if WIN32: socketclosename = 'closesocket' diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -589,3 +589,15 @@ return 0 fc = compile(f, [], thread=True) assert fc() == 0 + +def test_socket_saves_errno(tmpdir): + # ensure errno is set to a known value... + unconnected_sock = RSocket() + e = py.test.raises(CSocketError, unconnected_sock.recv, 1024) + # ...which is ENOTCONN + assert e.value.errno == errno.ENOTCONN + + e = py.test.raises(CSocketError, + RSocket, + family=AF_INET, type=SOCK_STREAM, proto=SOL_UDP) + assert e.value.errno in (errno.EPROTOTYPE, errno.EPROTONOSUPPORT) From pypy.commits at gmail.com Tue Jun 28 08:47:23 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:23 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: nonsense Message-ID: <577271db.85c11c0a.1287f.ffffc683@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85429:eedbe90f19b9 Date: 2016-06-24 18:15 +0200 http://bitbucket.org/pypy/pypy/changeset/eedbe90f19b9/ Log: nonsense diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -355,9 +355,10 @@ if type == 'i': return NotVirtualStateInfoInt(cpu, type, info) if type == 'r': - assert isinstance(info, PtrInfo) - if info is not None and info._compatibility_conditions is not None: - return NotVirtualStateInfoPtrCompatible(cpu, type, info) + if info is not None: + if (isinstance(info, PtrInfo) and + info._compatibility_conditions is not None): + return NotVirtualStateInfoPtrCompatible(cpu, type, info) return NotVirtualStateInfoPtr(cpu, type, info) return NotVirtualStateInfo(cpu, type, info) From pypy.commits at gmail.com Tue Jun 28 08:47:18 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:18 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: translation fixes Message-ID: <577271d6.d11b1c0a.15db7.ffffb8f9@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85427:c1d4078e392b Date: 2016-06-24 17:31 +0200 http://bitbucket.org/pypy/pypy/changeset/c1d4078e392b/ Log: translation fixes diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -88,6 +88,7 @@ if preamble_info._compatibility_conditions: info_in_loop = op.get_forwarded() if info_in_loop is not None: + assert isinstance(info_in_loop, info.PtrInfo) ccond = preamble_info._compatibility_conditions ccond = ccond.frozen_copy() info_in_loop._compatibility_conditions = ccond diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -1,7 +1,7 @@ from rpython.jit.metainterp.walkvirtual import VirtualVisitor from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\ - ArrayStructInfo, AbstractStructPtrInfo + ArrayStructInfo, AbstractStructPtrInfo, PtrInfo from rpython.jit.metainterp.optimizeopt.intutils import \ MININT, MAXINT, IntBound, IntLowerBound from rpython.jit.metainterp.resoperation import rop, ResOperation,\ @@ -355,6 +355,7 @@ if type == 'i': return NotVirtualStateInfoInt(cpu, type, info) if type == 'r': + assert isinstance(info, PtrInfo) if info is not None and info._compatibility_conditions is not None: return NotVirtualStateInfoPtrCompatible(cpu, type, info) return NotVirtualStateInfoPtr(cpu, type, info) From pypy.commits at gmail.com Tue Jun 28 08:47:25 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:25 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: corner case: sometimes the guarded value is really missing from the failargs, Message-ID: <577271dd.c4cb1c0a.4ebc4.ffff953a@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85430:f8abbe802a80 Date: 2016-06-24 23:39 +0200 http://bitbucket.org/pypy/pypy/changeset/f8abbe802a80/ Log: corner case: sometimes the guarded value is really missing from the failargs, in which case we are annoyed, but should still not crash. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1114,19 +1114,19 @@ # if new_loop starts with another guard_compatible on the same argument # (which is most of the time) we have to connect the new guard's descr # to this descr - assert self.failarg_index != -1 - arg = new_loop.inputargs[self.failarg_index] - firstop = new_loop.operations[0] compat_cond = None - if (firstop.getopnum() == rop.GUARD_COMPATIBLE and - firstop.getarg(0) is arg): - # a guard_compatible about the same box - # remove it, it doesn't have to be checked in the bridge - del new_loop.operations[0] - newdescr = firstop.getdescr() - assert isinstance(newdescr, GuardCompatibleDescr) - compat_cond = newdescr._compatibility_conditions - self.other_compat_conditions.append(compat_cond) + if self.failarg_index != -1: + arg = new_loop.inputargs[self.failarg_index] + firstop = new_loop.operations[0] + if (firstop.getopnum() == rop.GUARD_COMPATIBLE and + firstop.getarg(0) is arg): + # a guard_compatible about the same box + # remove it, it doesn't have to be checked in the bridge + del new_loop.operations[0] + newdescr = firstop.getdescr() + assert isinstance(newdescr, GuardCompatibleDescr) + compat_cond = newdescr._compatibility_conditions + self.other_compat_conditions.append(compat_cond) asminfo = ResumeGuardDescr.compile_and_attach( self, metainterp, new_loop, orig_inputargs) if compat_cond: @@ -1134,8 +1134,12 @@ return asminfo def make_a_counter_per_value(self, guard_value_op, index): - self.failarg_index = guard_value_op.getfailargs().index( - guard_value_op.getarg(0)) + try: + self.failarg_index = guard_value_op.getfailargs().index( + guard_value_op.getarg(0)) + except ValueError: + pass # we don't set the failarg_index, too bad + # this is not actually enabling the counter_per_value logic, # which right now gives bad results with a GUARD_COMPATIBLE #ResumeGuardDescr.make_a_counter_per_value(self, guard_value_op, index) diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -362,6 +362,53 @@ # trace, two bridges, a finish bridge self.check_trace_count(4) + def test_merge_obj_not_in_failargs(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + p1 = lltype.malloc(S) + p1.x = 1 + + p2 = lltype.malloc(S) + p2.x = 1 + + driver = jit.JitDriver(greens = [], reds = ['n']) + + class A(object): + pass + + c = A() + c.count = 0 + @jit.elidable_compatible() + def g(s, ignored): + c.count += 1 + return s.x + + class B(object): + pass + + glob_b = B() + + def f(n): + while n > 0: + driver.can_enter_jit(n=n) + driver.jit_merge_point(n=n) + x = glob_b.x + n -= g(x, "abc") + if n & 2: + n -= 2 + + def main(): + g(p1, "def") # make annotator not make argument constant + glob_b.x = p1 + f(1000) + glob_b.x = p2 + f(1000) + return c.count + + x = self.meta_interp(main, []) + + # trace, two bridges, a finish bridge + self.check_trace_count(4) + def test_merge_switch_object(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) p1 = lltype.malloc(S) From pypy.commits at gmail.com Tue Jun 28 08:47:28 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:28 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: typo Message-ID: <577271e0.9a4a1c0a.21a57.ffff9726@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85431:3e20ab453a7c Date: 2016-06-27 14:55 +0200 http://bitbucket.org/pypy/pypy/changeset/3e20ab453a7c/ Log: typo diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -293,7 +293,7 @@ else: assert rettype == VOID # XXX maybe we should forbid this - call_op = ResOperation(rop.CALL_PURE_R, args, descr) + call_op = ResOperation(rop.CALL_PURE_N, args, descr) short.append(call_op) return # add result to call_pure_results From pypy.commits at gmail.com Tue Jun 28 08:47:31 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:31 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: this test shows a problem: we now get 88 bridges(!) Message-ID: <577271e3.42431c0a.e4d84.ffffc23f@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85433:cedd97e275af Date: 2016-06-28 14:46 +0200 http://bitbucket.org/pypy/pypy/changeset/cedd97e275af/ Log: this test shows a problem: we now get 88 bridges(!) diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -370,6 +370,12 @@ p2 = lltype.malloc(S) p2.x = 1 + p3 = lltype.malloc(S) + p3.x = 2 + + p4 = lltype.malloc(S) + p4.x = 2 + driver = jit.JitDriver(greens = [], reds = ['n']) class A(object): @@ -402,6 +408,12 @@ f(1000) glob_b.x = p2 f(1000) + glob_b.x = p3 + f(1000) + glob_b.x = p4 + f(1000) + glob_b.x = p1 + f(1000) return c.count x = self.meta_interp(main, []) From pypy.commits at gmail.com Tue Jun 28 08:47:29 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 05:47:29 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: simply call @elidable_compatible functions where we don't have a result Message-ID: <577271e1.24adc20a.d9c46.ffffb931@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85432:f9d5045ab715 Date: 2016-06-27 19:10 +0200 http://bitbucket.org/pypy/pypy/changeset/f9d5045ab715/ Log: simply call @elidable_compatible functions where we don't have a result (this can only happen when inlining the short preamble into a bridge) diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -78,6 +78,20 @@ return True def prepare_const_arg_call(self, op, optimizer): + copied_op, cond = self._prepare_const_arg_call(op, optimizer) + if copied_op: + result = optimizer._can_optimize_call_pure(copied_op) + if result is None: + # just call it, we can do that with an @elidable_compatible + # function + result = do_call( + optimizer.cpu, copied_op.getarglist(), + copied_op.getdescr()) + return copied_op, cond, result + else: + return None, None, None + + def _prepare_const_arg_call(self, op, optimizer): from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr # replace further arguments by constants, if the optimizer knows them # already diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -142,17 +142,15 @@ ccond = info._compatibility_conditions if ccond: # it's subject to guard_compatible - copied_op, cond = ccond.prepare_const_arg_call( + copied_op, cond, result = ccond.prepare_const_arg_call( op, self.optimizer) if copied_op: - result = self._can_optimize_call_pure(copied_op) - if result is not None: - recorded = ccond.record_condition( - cond, result, self.optimizer) - if recorded: - self.make_constant(op, result) - self.last_emitted_operation = REMOVED - return + recorded = ccond.record_condition( + cond, result, self.optimizer) + if recorded: + self.make_constant(op, result) + self.last_emitted_operation = REMOVED + return # Step 1: check if all arguments are constant for arg in op.getarglist(): diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -528,3 +528,78 @@ self.check_trace_count(7) + def test_quasi_immutable_merge_short_preamble(self): + from rpython.rlib.objectmodel import we_are_translated + class C(object): + _immutable_fields_ = ['version?'] + + class Version(object): + def __init__(self, cls): + self.cls = cls + p1 = C() + p1.version = Version(p1) + p1.x = 1 + p2 = C() + p2.version = Version(p2) + p2.x = 1 + p3 = C() + p3.version = Version(p3) + p3.x = 3 + + driver = jit.JitDriver(greens = [], reds = ['n']) + + class Counter(object): + pass + + c = Counter() + c.count = 0 + @jit.elidable_compatible() + def g(cls, v): + if we_are_translated(): + c.count += 1 + return cls.x + + class B(object): + pass + + glob_b = B() + + def f(n, x): + glob_b.x = x + res = 0 + while n > 0: + driver.can_enter_jit(n=n) + driver.jit_merge_point(n=n) + x = jit.hint(glob_b.x, promote_compatible=True) + v = x.version + res = g(x, v) + n -= res + if n % 11 == 5: + n -= 1 + return res + + def main(x): + res = f(100, p1) + assert res == 1 + res = f(100, p2) + assert res == 1 + res = f(100, p3) + assert res == 3 + main(True) + main(False) + + x = self.meta_interp(main, [True]) + assert x < 70 + x = self.meta_interp(main, [True]) + assert x < 70 + x = self.meta_interp(main, [True]) + assert x < 70 + x = self.meta_interp(main, [True]) + assert x < 70 + + x = self.meta_interp(main, [False]) + assert x < 70 + self.check_trace_count(9) + self.check_resops(call_i=0) + + From pypy.commits at gmail.com Tue Jun 28 10:25:15 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 28 Jun 2016 07:25:15 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: finish the impl. of vec_float_eq/ne + test Message-ID: <577288cb.c72d1c0a.a8f8b.137d@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85434:780c12936a64 Date: 2016-06-28 16:24 +0200 http://bitbucket.org/pypy/pypy/changeset/780c12936a64/ Log: finish the impl. of vec_float_eq/ne + test diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -683,9 +683,12 @@ vor = VX(4, XO8=1156) veqv = VX(4, XO8=1668) vxor = VX(4, XO8=1220) + vnor = VX(4, XO8=1284) # vector move register is alias to vector or vmr = vor + # complement is equivalent to vnor + vnot = vnor # shift, perm and select lvsl = XV(31, XO1=6) diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py --- a/rpython/jit/backend/ppc/runner.py +++ b/rpython/jit/backend/ppc/runner.py @@ -52,6 +52,7 @@ self.vector_ext = AltiVectorExt() self.vector_extension = True # ??? self.vector_horizontal_operations = True + self.assembler.setup_once_vector() @rgc.no_release_gil def finish_once(self): diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -16,6 +16,8 @@ from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET import rpython.jit.backend.ppc.register as r import rpython.jit.backend.ppc.condition as c +from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper +from rpython.rtyper.lltypesystem import lltype, rffi def not_implemented(msg): msg = '[ppc/vector_ext] %s\n' % msg @@ -23,12 +25,54 @@ llop.debug_print(lltype.Void, msg) raise NotImplementedError(msg) +def flush_vec_cc(asm, regalloc, condition, size, result_loc): + # After emitting an instruction that leaves a boolean result in + # a condition code (cc), call this. In the common case, result_loc + # will be set to SPP by the regalloc, which in this case means + # "propagate it between this operation and the next guard by keeping + # it in the cc". In the uncommon case, result_loc is another + # register, and we emit a load from the cc into this register. + assert asm.guard_success_cc == c.cond_none + if result_loc is r.SPP: + asm.guard_success_cc = condition + else: + resval = result_loc.value + # either doubleword integer 1 (2x) or word integer 1 (4x) + ones = regalloc.ivrm.get_scratch_reg().value + zeros = regalloc.ivrm.get_scratch_reg().value + asm.mc.vxor(zeros, zeros, zeros) + if size == 4: + asm.mc.vspltisw(ones, 1) + else: + assert size == 8 + tloc = regalloc.rm.get_scratch_reg() + asm.mc.load_imm(tloc, asm.VEC_DOUBLE_WORD_ONES) + asm.mc.lvx(ones, 0, tloc.value) + asm.mc.vsel(resval, zeros, ones, resval) + class AltiVectorExt(VectorExt): pass class VectorAssembler(object): _mixin_ = True + VEC_DOUBLE_WORD_ONES = 0 + + def setup_once_vector(self): + if IS_BIG_ENDIAN: + # 2x 64 bit signed integer(1) BE + data = (b'\x00' * 7 + b'\x01') * 2 + else: + # 2x 64 bit signed integer(1) LE + data = (b'\x01' + b'\x00' * 7) * 2 + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + mem = datablockwrapper.malloc_aligned(len(data), alignment=16) + datablockwrapper.done() + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), mem) + for i in range(len(data)): + addr[i] = data[i] + self.VEC_DOUBLE_WORD_ONES = mem + def emit_vec_load_f(self, op, arglocs, regalloc): resloc, baseloc, indexloc, size_loc, ofs, integer_loc, aligned_loc = arglocs #src_addr = addr_add(baseloc, ofs_loc, ofs.value, 0) @@ -360,6 +404,7 @@ else: notimplemented("[ppc/assembler] float == for size %d" % size) self.mc.lvx(resloc.value, off, r.SP.value) + flush_vec_cc(self, regalloc, c.EQ, op.bytesize, resloc) def emit_vec_float_ne(self, op, arglocs, regalloc): resloc, loc1, loc2, sizeloc = arglocs @@ -371,15 +416,16 @@ self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET) if size == 4: self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value) - self.mc.xxlandc(tmp, tmp, tmp) # negate self.mc.stxvw4x(tmp, off, r.SP.value) elif size == 8: self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value) - self.mc.xxlandc(tmp, tmp, tmp) # negate self.mc.stxvd2x(tmp, off, r.SP.value) else: notimplemented("[ppc/assembler] float == for size %d" % size) - self.mc.lvx(resloc.value, off, r.SP.value) + res = resloc.value + self.mc.lvx(res, off, r.SP.value) + self.mc.vnor(res, res, res) # complement + flush_vec_cc(self, regalloc, c.NE, op.bytesize, resloc) def emit_vec_cast_int_to_float(self, op, arglocs, regalloc): res, l0 = arglocs diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -185,6 +185,8 @@ test_vec_float_cmp_eq = \ vec_float_binary(lambda a,b: a == b, rffi.DOUBLE) + test_vec_float_cmp_ne = \ + vec_float_binary(lambda a,b: a != b, rffi.DOUBLE) def _vector_simple_int(self, func, type, data): func = always_inline(func) From pypy.commits at gmail.com Tue Jun 28 11:15:05 2016 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 28 Jun 2016 08:15:05 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: add the backend functions for vec_int_eq/ne + test, Message-ID: <57729479.c5ddc20a.3320.10b6@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85435:a404127ebe68 Date: 2016-06-28 17:14 +0200 http://bitbucket.org/pypy/pypy/changeset/a404127ebe68/ Log: add the backend functions for vec_int_eq/ne + test, implement vec_expand_f, test passes that uses vec_expand_f diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -160,7 +160,7 @@ factor, offset): factor, offset, new_index_box = cpu_simplify_scale(self.cpu, index_box, factor, offset) if index_box is not new_index_box: - self.emit_op(index_box) + self.emit_op(new_index_box) return factor, offset, new_index_box def emit_gc_load_or_indexed(self, op, ptr_box, index_box, itemsize, diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py --- a/rpython/jit/backend/llsupport/vector_ext.py +++ b/rpython/jit/backend/llsupport/vector_ext.py @@ -1,3 +1,4 @@ +from rpython.jit.backend.llsupport.rewrite import cpu_simplify_scale from rpython.jit.backend.llsupport.descr import (unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr) from rpython.rlib.objectmodel import specialize, always_inline @@ -130,7 +131,7 @@ if opnum in (rop.SETARRAYITEM_GC, rop.SETARRAYITEM_RAW): itemsize, basesize, _ = unpack_arraydescr(op.getdescr()) index_box = op.getarg(1) - _, _, changed = cpu_simplify_scale(index_box, itemsize, basesize) + _, _, changed = cpu_simplify_scale(state.cpu, index_box, itemsize, basesize) if changed is not index_box: state.oplist.append(changed) op.setarg(1, changed) diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -16,8 +16,10 @@ from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET import rpython.jit.backend.ppc.register as r import rpython.jit.backend.ppc.condition as c +import rpython.jit.backend.ppc.locations as l from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.jit.codewriter import longlong def not_implemented(msg): msg = '[ppc/vector_ext] %s\n' % msg @@ -441,13 +443,14 @@ res, l0, l1, sizeloc = arglocs size = sizeloc.value if size == 1: - self.vcmpequbx(res.value, l0.value, l1.value) + self.mc.vcmpequbx(res.value, l0.value, l1.value) elif size == 2: - self.vcmpequhx(res.value, l0.value, l1.value) + self.mc.vcmpequhx(res.value, l0.value, l1.value) elif size == 4: - self.vcmpequwx(res.value, l0.value, l1.value) + self.mc.vcmpequwx(res.value, l0.value, l1.value) elif size == 8: - self.vcmpequdx(res.value, l0.value, l1.value) + self.mc.vcmpequdx(res.value, l0.value, l1.value) + flush_vec_cc(self, regalloc, c.EQ, op.bytesize, res) def emit_vec_int_ne(self, op, arglocs, regalloc): res, l0, l1, sizeloc = arglocs @@ -455,62 +458,55 @@ tmp = regalloc.get_scratch_reg().value self.mc.vxor(tmp, tmp, tmp) if size == 1: - self.vcmpequb(res.value, l0.value, l1.value) - self.vcmpequbx(res.value, res.value, tmp) + self.mc.vcmpequbx(res.value, res.value, tmp) elif size == 2: - self.vcmpequh(res.value, l0.value, l1.value) - self.vcmpequhx(res.value, res.value, tmp) + self.mc.vcmpequhx(res.value, res.value, tmp) elif size == 4: - self.vcmpequw(res.value, l0.value, l1.value) - self.vcmpequwx(res.value, res.value, tmp) + self.mc.vcmpequwx(res.value, res.value, tmp) elif size == 8: - self.vcmpequd(res.value, l0.value, l1.value) - self.vcmpequdx(res.value, res.value, tmp) + self.mc.vcmpequdx(res.value, res.value, tmp) + self.mc.vnor(res.value, res.value, res.value) + flush_vec_cc(self, regalloc, c.NE, op.bytesize, res) - #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): - # self.mc.CVTPD2DQ(resloc, arglocs[0]) - #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): - # self.mc.CVTPS2PD(resloc, arglocs[0]) + def emit_vec_expand_f(self, op, arglocs, regalloc): + resloc, srcloc = arglocs + size = op.bytesize + res = resloc.value + if isinstance(srcloc, l.ConstFloatLoc): + # they are aligned! + assert size == 8 + tloc = regalloc.rm.get_scratch_reg() + self.mc.load_imm(tloc, srcloc.value) + self.mc.lxvd2x(res, 0, tloc.value) + elif size == 8: + self.mc.vmr(res, srcloc.value, srcloc.value) + else: + notimplemented("[ppc/assembler] vec expand in this combination not supported") - #def genop_vec_expand_f(self, op, arglocs, regalloc): - # srcloc, sizeloc = arglocs - # size = sizeloc.value - # if isinstance(srcloc, ConstFloatLoc): - # # they are aligned! - # self.mc.MOVAPD(resloc, srcloc) - # elif size == 4: - # # the register allocator forces src to be the same as resloc - # # r = (s[0], s[0], r[0], r[0]) - # # since resloc == srcloc: r = (r[0], r[0], r[0], r[0]) - # self.mc.SHUFPS_xxi(resloc.value, srcloc.value, 0) - # elif size == 8: - # self.mc.MOVDDUP(resloc, srcloc) - # else: - # raise AssertionError("float of size %d not supported" % (size,)) - - #def genop_vec_expand_i(self, op, arglocs, regalloc): - # srcloc, sizeloc = arglocs - # if not isinstance(srcloc, RegLoc): - # self.mov(srcloc, X86_64_SCRATCH_REG) - # srcloc = X86_64_SCRATCH_REG - # assert not srcloc.is_xmm - # size = sizeloc.value - # if size == 1: - # self.mc.PINSRB_xri(resloc.value, srcloc.value, 0) - # self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr)) - # elif size == 2: - # self.mc.PINSRW_xri(resloc.value, srcloc.value, 0) - # self.mc.PINSRW_xri(resloc.value, srcloc.value, 4) - # self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0) - # self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0) - # elif size == 4: - # self.mc.PINSRD_xri(resloc.value, srcloc.value, 0) - # self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0) - # elif size == 8: - # self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0) - # self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1) - # else: - # raise AssertionError("cannot handle size %d (int expand)" % (size,)) + def emit_vec_expand_i(self, op, arglocs, regalloc): + notimplemented("[vec expand i]") + srcloc, sizeloc = arglocs + if not isinstance(srcloc, RegLoc): + self.mov(srcloc, X86_64_SCRATCH_REG) + srcloc = X86_64_SCRATCH_REG + assert not srcloc.is_xmm + size = sizeloc.value + if size == 1: + self.mc.PINSRB_xri(resloc.value, srcloc.value, 0) + self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr)) + elif size == 2: + self.mc.PINSRW_xri(resloc.value, srcloc.value, 0) + self.mc.PINSRW_xri(resloc.value, srcloc.value, 4) + self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0) + self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0) + elif size == 4: + self.mc.PINSRD_xri(resloc.value, srcloc.value, 0) + self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0) + elif size == 8: + self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0) + self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1) + else: + raise AssertionError("cannot handle size %d (int expand)" % (size,)) #def genop_vec_pack_i(self, op, arglocs, regalloc): # resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs @@ -622,6 +618,12 @@ #genop_vec_unpack_f = genop_vec_pack_f + # needed as soon as PPC's support_singlefloat is implemented! + #def genop_vec_cast_float_to_int(self, op, arglocs, regalloc): + # self.mc.CVTPD2DQ(resloc, arglocs[0]) + #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc): + # self.mc.CVTPS2PD(resloc, arglocs[0]) + class VectorRegalloc(object): _mixin_ = True @@ -789,28 +791,24 @@ #prepare_vec_unpack_f = prepare_vec_unpack_i - #def prepare_vec_expand_f(self, op): - # assert isinstance(op, VectorOp) - # arg = op.getarg(0) - # args = op.getarglist() - # if arg.is_constant(): - # resloc = self.xrm.force_allocate_reg(op) - # srcloc = self.xrm.expand_float(op.bytesize, arg) - # else: - # resloc = self.xrm.force_result_in_reg(op, arg, args) - # srcloc = resloc - # self.perform(op, [srcloc, imm(op.bytesize)], resloc) + def expand_float(self, size, box): + adr = self.assembler.datablockwrapper.malloc_aligned(16, 16) + fs = box.getfloatstorage() + rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = fs + rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = fs + return l.ConstFloatLoc(adr) - #def prepare_vec_expand_i(self, op): - # assert isinstance(op, VectorOp) - # arg = op.getarg(0) - # args = op.getarglist() - # if arg.is_constant(): - # srcloc = self.rm.convert_to_imm(arg) - # else: - # srcloc = self.make_sure_var_in_reg(arg, args) - # resloc = self.xrm.force_allocate_reg(op, args) - # self.perform(op, [srcloc, imm(op.bytesize)], resloc) + def prepare_vec_expand_f(self, op): + arg = op.getarg(0) + if arg.is_constant(): + l0 = self.expand_float(op.bytesize, arg) + res = self.force_allocate_vector_reg(op) + else: + l0 = self.ensure_vector_reg(arg) + res = self.force_allocate_vector_reg(op) + return [res, l0] + + prepare_vec_expand_i = prepare_vec_expand_f def prepare_vec_int_is_true(self, op): arg = op.getarg(0) @@ -819,12 +817,12 @@ resloc = self.force_allocate_vector_reg(op) return [resloc, argloc, imm(arg.bytesize)] - #def _prepare_vec(self, op): - # # pseudo instruction, needed to create a new variable - # self.xrm.force_allocate_reg(op) + def _prepare_vec(self, op): + # pseudo instruction, needed to allocate a register for a new variable + return [self.force_allocate_vector_reg(op)] - #prepare_vec_i = _prepare_vec - #prepare_vec_f = _prepare_vec + prepare_vec_i = _prepare_vec + prepare_vec_f = _prepare_vec def prepare_vec_cast_float_to_int(self, op): l0 = self.ensure_vector_reg(op.getarg(0)) @@ -832,18 +830,15 @@ return [res, l0] prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int - #prepare_vec_cast_float_to_singlefloat = prepare_vec_cast_float_to_int - #prepare_vec_cast_singlefloat_to_float = prepare_vec_cast_float_to_int - #def prepare_vec_guard_true(self, op): - # arg = op.getarg(0) - # loc = self.loc(arg) - # self.assembler.guard_vector(op, self.loc(arg), True) - # self.perform_guard(op, [], None) + def _prepare_guard(self, box): + if self.assembler.guard_success_cc == c.cond_none: + notimplemented("[ppc/regalloc] guard") + self.assembler.guard_success_cc = c.NE - #def prepare_vec_guard_false(self, op): - # arg = op.getarg(0) - # loc = self.loc(arg) - # self.assembler.guard_vector(op, self.loc(arg), False) - # self.perform_guard(op, [], None) + def prepare_vec_guard_true(self, op): + self._prepare_guard(op.getarg(0)) + def prepare_vec_guard_false(self, op): + self._prepare_guard(op.getarg(0)) + diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -265,6 +265,11 @@ test_vec_xor_short = \ vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT) + test_vec_int_eq = \ + vec_int_arith(lambda a,b: a == b, rffi.SIGNED) + test_vec_int_ne = \ + vec_int_arith(lambda a,b: a == b, rffi.SIGNED) + @py.test.mark.parametrize('i',[1,2,3,4,9]) def test_vec_register_too_small_vector(self, i): myjitdriver = JitDriver(greens = [], From pypy.commits at gmail.com Tue Jun 28 11:38:07 2016 From: pypy.commits at gmail.com (rlamy) Date: Tue, 28 Jun 2016 08:38:07 -0700 (PDT) Subject: [pypy-commit] pypy default: Internal string->bytes renaming Message-ID: <577299df.50991c0a.c9293.ffffa7fe@mx.google.com> Author: Ronan Lamy Branch: Changeset: r85436:4b3a2a0269e2 Date: 2016-06-28 16:37 +0100 http://bitbucket.org/pypy/pypy/changeset/4b3a2a0269e2/ Log: Internal string->bytes renaming diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -11,7 +11,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject ## -## Implementation of PyStringObject +## Implementation of PyBytesObject ## ================================ ## ## The problem @@ -29,15 +29,15 @@ ## Solution ## -------- ## -## PyStringObject contains two additional members: the ob_size and a pointer to a +## PyBytesObject contains two additional members: the ob_size and a pointer to a ## char ob_sval; it may be NULL. ## -## - A string allocated by pypy will be converted into a PyStringObject with a +## - A string allocated by pypy will be converted into a PyBytesObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is ## allocated (with flavor='raw') and content is copied. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyStringObject structure, and a buffer with the specified +## allocate a PyBytesObject structure, and a buffer with the specified ## size+1, but the reference won't be stored in the global map; there is no ## corresponding object in pypy. When from_ref() or Py_INCREF() is called, ## the pypy string is created, and added to the global map of tracked @@ -55,58 +55,58 @@ ## corresponds to the pypy gc-managed string. ## -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyVarObjectFields + \ +PyBytesObjectStruct = lltype.ForwardReference() +PyBytesObject = lltype.Ptr(PyBytesObjectStruct) +PyBytesObjectFields = PyVarObjectFields + \ (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +cpython_struct("PyStringObject", PyBytesObjectFields, PyBytesObjectStruct) @bootstrap_function -def init_stringobject(space): - "Type description of PyStringObject" +def init_bytesobject(space): + "Type description of PyBytesObject" make_typedescr(space.w_str.layout.typedef, - basestruct=PyStringObject.TO, - attach=string_attach, - dealloc=string_dealloc, - realize=string_realize) + basestruct=PyBytesObject.TO, + attach=bytes_attach, + dealloc=bytes_dealloc, + realize=bytes_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocate a PyStringObject and its ob_sval, but without a corresponding - interpreter object. The ob_sval may be mutated, until string_realize() is + Allocate a PyBytesObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until bytes_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) py_obj = typedescr.allocate(space, space.w_str, length) - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str -def string_attach(space, py_obj, w_obj): +def bytes_attach(space, py_obj, w_obj): """ - Copy RPython string object contents to a PyStringObject. The + Copy RPython string object contents to a PyBytesObject. The c_ob_sval must not be modified. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = space.str_w(w_obj) if py_str.c_ob_size < len(s): raise oefmt(space.w_ValueError, - "string_attach called on object with ob_size %d but trying to store %d", - py_str.c_ob_size, len(s)) + "bytes_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL -def string_realize(space, py_obj): +def bytes_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject ob_sval must not + Creates the string in the interpreter. The PyBytesObject ob_sval must not be modified after this call. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(W_BytesObject, w_type) @@ -117,8 +117,8 @@ return w_obj @cpython_api([PyObject], lltype.Void, header=None) -def string_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. +def bytes_dealloc(space, py_obj): + """Frees allocated PyBytesObject resources. """ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -154,10 +154,10 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) + ref_str = rffi.cast(PyBytesObject, ref) if not pyobj_has_w_obj(ref): # XXX Force the ref? - string_realize(space, ref) + bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) @@ -166,7 +166,7 @@ # if no w_str is associated with this ref, # return the c-level ptr as RW if not pyobj_has_w_obj(ref): - py_str = rffi.cast(PyStringObject, ref) + py_str = rffi.cast(PyBytesObject, ref) return py_str.c_ob_sval return _PyString_AsString(space, ref) @@ -183,8 +183,8 @@ from_ref(space, ref)) if not pyobj_has_w_obj(ref): # force the ref - string_realize(space, ref) - ref_str = rffi.cast(PyStringObject, ref) + bytes_realize(space, ref) + ref_str = rffi.cast(PyBytesObject, ref) data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size @@ -200,7 +200,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: - ref = rffi.cast(PyStringObject, ref) + ref = rffi.cast(PyBytesObject, ref) return ref.c_ob_size else: w_obj = from_ref(space, ref) @@ -222,7 +222,7 @@ if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") - py_str = rffi.cast(PyStringObject, ref[0]) + py_str = rffi.cast(PyBytesObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -164,7 +164,7 @@ pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) if pytype.c_tp_itemsize != 0: - itemcount = space.len_w(w_obj) # PyStringObject and subclasses + itemcount = space.len_w(w_obj) # PyBytesObject and subclasses else: itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -38,7 +38,7 @@ @cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. + """Frees allocated PyBytesObject resources. """ py_slice = rffi.cast(PySliceObject, py_obj) Py_DecRef(space, py_slice.c_start) @@ -73,7 +73,7 @@ length length, and store the length of the slice in slicelength. Out of bounds indices are clipped in a manner consistent with the handling of normal slices. - + Returns 0 on success and -1 on error with exception set.""" if not PySlice_Check(space, w_slice): PyErr_BadInternalCall(space) @@ -88,11 +88,11 @@ """Retrieve the start, stop and step indices from the slice object slice, assuming a sequence of length length. Treats indices greater than length as errors. - + Returns 0 on success and -1 on error with no exception set (unless one of the indices was not None and failed to be converted to an integer, in which case -1 is returned with an exception set). - + You probably do not want to use this function. If you want to use slice objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.bytesobject import new_empty_str, PyStringObject +from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr @@ -82,14 +82,14 @@ """ PyObject *base; PyTypeObject * type; - PyStringObject *obj; + PyBytesObject *obj; base = PyBytes_FromString("test"); if (PyBytes_GET_SIZE(base) != 4) return PyLong_FromLong(-PyBytes_GET_SIZE(base)); type = base->ob_type; if (type->tp_itemsize != 1) return PyLong_FromLong(type->tp_itemsize); - obj = (PyStringObject*)type->tp_alloc(type, 10); + obj = (PyBytesObject*)type->tp_alloc(type, 10); if (PyBytes_GET_SIZE(obj) != 10) return PyLong_FromLong(PyBytes_GET_SIZE(obj)); /* cannot work, there is only RO access @@ -266,7 +266,7 @@ PyObject *s = args; Py_INCREF(s); PyString_InternInPlace(&s); - if (((PyStringObject*)s)->ob_sstate == SSTATE_NOT_INTERNED) + if (((PyBytesObject*)s)->ob_sstate == SSTATE_NOT_INTERNED) { Py_DECREF(s); s = PyString_FromString("interned error"); @@ -284,7 +284,7 @@ ("test_macro_invocations", "METH_NOARGS", """ PyObject* o = PyString_FromString(""); - PyStringObject* u = (PyStringObject*)o; + PyBytesObject* u = (PyBytesObject*)o; PyString_GET_SIZE(u); PyString_GET_SIZE(o); @@ -301,17 +301,17 @@ ("test_hash", "METH_VARARGS", ''' PyObject* obj = (PyTuple_GetItem(args, 0)); - long hash = ((PyStringObject*)obj)->ob_shash; + long hash = ((PyBytesObject*)obj)->ob_shash; return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - /*int sstate = ((PyStringObject*)s)->ob_sstate; + /*int sstate = ((PyBytesObject*)s)->ob_sstate; printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - /*sstate = ((PyStringObject*)s)->ob_sstate; + /*sstate = ((PyBytesObject*)s)->ob_sstate; printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); @@ -345,7 +345,7 @@ PyObject_HEAD_INIT(NULL) 0, /* ob_size */ "bar.string_", /* tp_name*/ - sizeof(PyStringObject), /* tp_basicsize*/ + sizeof(PyBytesObject), /* tp_basicsize*/ 0 /* tp_itemsize */ }; @@ -401,7 +401,7 @@ return NULL; } destptr = PyString_AS_STRING(obj); - ((PyStringObject *)obj)->ob_shash = -1; + ((PyBytesObject *)obj)->ob_shash = -1; memcpy(destptr, data, itemsize); return obj; } @@ -429,14 +429,14 @@ py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 3) - py_str = rffi.cast(PyStringObject, ar[0]) + py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 3 assert py_str.c_ob_sval[1] == 'b' assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 10) - py_str = rffi.cast(PyStringObject, ar[0]) + py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 10 assert py_str.c_ob_sval[1] == 'b' assert py_str.c_ob_sval[10] == '\x00' diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -167,7 +167,7 @@ py_memberdescr.c_d_member = w_obj.member def memberdescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? member = rffi.cast(lltype.Ptr(PyMemberDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_MemberDescr, w_type) @@ -192,7 +192,7 @@ py_methoddescr.c_d_method = w_obj.ml def classmethoddescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? method = rffi.cast(lltype.Ptr(PyMethodDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_PyCClassMethodObject, w_type) @@ -201,7 +201,7 @@ return w_obj def methoddescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? method = rffi.cast(lltype.Ptr(PyMethodDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_PyCMethodObject, w_type) @@ -566,7 +566,7 @@ if space.is_w(w_type, space.w_str): # Special case: str doesn't support get_raw_address(), so we have a # custom get*buffer that instead gives the address of the char* in the - # PyStringObject*! + # PyBytesObject*! c_buf.c_bf_getreadbuffer = llhelper( str_getreadbuffer.api_func.functype, str_getreadbuffer.api_func.get_wrapper(space)) @@ -782,7 +782,7 @@ if py_type.c_ob_type: w_metatype = from_ref(space, rffi.cast(PyObject, py_type.c_ob_type)) - else: + else: # Somehow the tp_base type is created with no ob_type, notably # PyString_Type and PyBaseString_Type # While this is a hack, cpython does it as well. @@ -797,10 +797,10 @@ # inheriting tp_as_* slots base = py_type.c_tp_base if base: - if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number - if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence - if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping - if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer + if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number + if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence + if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping + if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer return w_obj From pypy.commits at gmail.com Tue Jun 28 12:07:18 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 09:07:18 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fix the problem Message-ID: <5772a0b6.cdcf1c0a.1df2.71e4@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85437:09617ecb9107 Date: 2016-06-28 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/09617ecb9107/ Log: fix the problem when attaching a new bridge to a GuardCompatibleDescr and we cannot find out whether the new bridge is checking conditions on the same variable as the GuardCompatibleDescr, the guard needs to henceforth always jump to that new trace if none of the other traces match. Otherwise we will keep compiling more and more code. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1091,6 +1091,12 @@ # list of compatibility conditions about the same variable, with # bridges attached to them self.other_compat_conditions = [] + # fallback jump target is the one to jump to if the generated bridges + # do not start with a guard_compatible at all, so we don't have a basis + # to decide on + # XXX it would be better to patch the guard properly in the backend, + # but later + self.fallback_jump_target = 0 def find_compatible(self, cpu, ref): """ callback for the CPU: given a value ref, it returns: @@ -1108,7 +1114,10 @@ if _compatibility_conditions.check_compat_and_activate( cpu, ref, self.rd_loop_token): return _compatibility_conditions.jump_target - return 0 + # none of the other conditions matched. if we have a + # fallback_jump_target, go there (otherwise we run the risk of + # producing arbitrary amounts of code) + return self.fallback_jump_target def compile_and_attach(self, metainterp, new_loop, orig_inputargs): # if new_loop starts with another guard_compatible on the same argument @@ -1131,6 +1140,9 @@ self, metainterp, new_loop, orig_inputargs) if compat_cond: compat_cond.jump_target = asminfo.asmaddr + else: + assert self.fallback_jump_target == 0 # this can never happen twice + self.fallback_jump_target = asminfo.asmaddr return asminfo def make_a_counter_per_value(self, guard_value_op, index): diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -418,8 +418,7 @@ x = self.meta_interp(main, []) - # trace, two bridges, a finish bridge - self.check_trace_count(4) + self.check_trace_count(6) def test_merge_switch_object(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -611,7 +610,7 @@ x = self.meta_interp(main, [False]) assert x < 70 - self.check_trace_count(9) + self.check_trace_count(7) self.check_resops(call_i=0) From pypy.commits at gmail.com Tue Jun 28 12:07:20 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 09:07:20 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fix tests Message-ID: <5772a0b8.cdcf1c0a.1df2.71e5@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85438:6215fed7ec26 Date: 2016-06-28 17:03 +0200 http://bitbucket.org/pypy/pypy/changeset/6215fed7ec26/ Log: fix tests diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -30,6 +30,9 @@ self._last_debug_merge_point = None self.quasi_immutable_deps = None + def _can_optimize_call_pure(self, op): + return True + class BaseTestGenerateGuards(BaseTest): def setup_class(self): classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) @@ -490,7 +493,7 @@ op = ResOperation( rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr)], descr=self.plaincalldescr) - copied_op, cond = ccond.prepare_const_arg_call( + copied_op, cond, result = ccond.prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) @@ -504,7 +507,7 @@ rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr), getfield_op], descr=self.nonwritedescr) - copied_op, cond = ccond.prepare_const_arg_call( + copied_op, cond, result = ccond.prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) value = info.PtrInfo() @@ -532,7 +535,9 @@ def test_virtualstate_guard_compatible(self): value1 = self.make_ccond_info() + ccond1 = value1._compatibility_conditions value2 = self.make_ccond_info() + ccond2 = value2._compatibility_conditions state1 = not_virtual(self.cpu, 'r', value1) state2 = not_virtual(self.cpu, 'r', value2) @@ -551,6 +556,7 @@ def test_virtualstate_guard_compatible_make_guards(self): value1 = self.make_ccond_info() value2 = self.make_ccond_info() + ccond2 = value2._compatibility_conditions state1 = not_virtual(self.cpu, 'r', value1) state2 = not_virtual(self.cpu, 'r', value2) @@ -559,7 +565,7 @@ cond = ccond2.conditions[:] ccond2.conditions = [cond[0]] box = InputArgRef(self.nodeaddr) - self.guards(state1, state2) + self.check_no_guards(state1, state2) ccond2.conditions = [cond[1]] self.check_no_guards(state1, state2) From pypy.commits at gmail.com Tue Jun 28 12:09:12 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 09:09:12 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: a comment Message-ID: <5772a128.e85dc20a.6fa1e.3de0@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85439:7ed893072cdd Date: 2016-06-28 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/7ed893072cdd/ Log: a comment diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1138,6 +1138,8 @@ self.other_compat_conditions.append(compat_cond) asminfo = ResumeGuardDescr.compile_and_attach( self, metainterp, new_loop, orig_inputargs) + # note that the backend will not patch the switch at all, so it is + # vital to *always* do something with asminfo.asmaddr if compat_cond: compat_cond.jump_target = asminfo.asmaddr else: From pypy.commits at gmail.com Tue Jun 28 12:09:14 2016 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 28 Jun 2016 09:09:14 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: merge default Message-ID: <5772a12a.56311c0a.8c973.ffffca45@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85440:790c399e5c6b Date: 2016-06-28 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/790c399e5c6b/ Log: merge default diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt --- a/pypy/doc/config/commandline.txt +++ b/pypy/doc/config/commandline.txt @@ -9,7 +9,7 @@ PyPy Python interpreter options ------------------------------- -The following options can be used after ``translate.py +The following options can be used after ``rpython targetpypystandalone`` or as options to ``py.py``. .. GENERATE: objspace @@ -22,7 +22,7 @@ General translation options --------------------------- -The following are options of ``translate.py``. They must be +The following are options of ``bin/rpython``. They must be given before the ``targetxxx`` on the command line. * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]` diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst --- a/pypy/doc/config/index.rst +++ b/pypy/doc/config/index.rst @@ -15,12 +15,12 @@ ./py.py <`objspace options`_> -and the ``translate.py`` translation entry +and the ``rpython/bin/rpython`` translation entry point which takes arguments of this form: .. parsed-literal:: - ./translate.py <`translation options`_> + ./rpython/bin/rpython <`translation options`_> For the common case of ```` being ``targetpypystandalone.py``, you can then pass the `object space options`_ after @@ -28,7 +28,7 @@ .. parsed-literal:: - ./translate.py <`translation options`_> targetpypystandalone.py <`objspace options`_> + ./rpython/bin/rpython <`translation options`_> targetpypystandalone.py <`objspace options`_> There is an `overview`_ of all command line arguments that can be passed in either position. diff --git a/pypy/doc/config/opt.rst b/pypy/doc/config/opt.rst --- a/pypy/doc/config/opt.rst +++ b/pypy/doc/config/opt.rst @@ -4,8 +4,8 @@ This meta-option selects a default set of optimization settings to use during a translation. Usage:: - translate.py --opt=# - translate.py -O# + bin/rpython --opt=# + bin/rpython -O# where ``#`` is the desired optimization level. The valid choices are: diff --git a/pypy/doc/config/translation.dont_write_c_files.txt b/pypy/doc/config/translation.dont_write_c_files.txt --- a/pypy/doc/config/translation.dont_write_c_files.txt +++ b/pypy/doc/config/translation.dont_write_c_files.txt @@ -1,4 +1,4 @@ write the generated C files to ``/dev/null`` instead of to the disk. Useful if -you want to use translate.py as a benchmark and don't want to access the disk. +you want to use translation as a benchmark and don't want to access the disk. .. _`translation documentation`: ../translation.html diff --git a/pypy/doc/config/translation.fork_before.txt b/pypy/doc/config/translation.fork_before.txt --- a/pypy/doc/config/translation.fork_before.txt +++ b/pypy/doc/config/translation.fork_before.txt @@ -1,4 +1,4 @@ This is an option mostly useful when working on the PyPy toolchain. If you use -it, translate.py will fork before the specified phase. If the translation +it, translation will fork before the specified phase. If the translation crashes after that fork, you can fix the bug in the toolchain, and continue translation at the fork-point. diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -122,7 +122,7 @@ $ hg up reflex-support # optional # This example shows python, but using pypy-c is faster and uses less memory - $ python rpython/translator/goal/translate.py --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy + $ python rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone --withmod-cppyy This will build a ``pypy-c`` that includes the cppyy module, and through that, Reflex support. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,8 @@ Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show the errno of the failing system call, but instead some random previous errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -400,7 +400,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -79,5 +79,5 @@ Py_DecRef(space, py_buf.c_b_base) else: rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -26,54 +26,10 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) PyByteArrayObjectFields = PyVarObjectFields -# (("ob_exports", rffi.INT), ("ob_alloc", rffi.LONG), ("ob_bytes", rffi.CCHARP)) cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) - at bootstrap_function -def init_bytearrayobject(space): - "Type description of PyByteArrayObject" - #make_typedescr(space.w_bytearray.layout.typedef, - # basestruct=PyByteArrayObject.TO, - # attach=bytearray_attach, - # dealloc=bytearray_dealloc, - # realize=bytearray_realize) - PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") -# XXX dead code to be removed -#def bytearray_attach(space, py_obj, w_obj): -# """ -# Fills a newly allocated PyByteArrayObject with the given bytearray object -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# py_ba.c_ob_size = len(space.str_w(w_obj)) -# py_ba.c_ob_bytes = lltype.nullptr(rffi.CCHARP.TO) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) - -#def bytearray_realize(space, py_obj): -# """ -# Creates the bytearray in the interpreter. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if not py_ba.c_ob_bytes: -# py_ba.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_ba.c_ob_size + 1, -# flavor='raw', zero=True) -# s = rffi.charpsize2str(py_ba.c_ob_bytes, py_ba.c_ob_size) -# w_obj = space.wrap(s) -# py_ba.c_ob_exports = rffi.cast(rffi.INT, 0) -# track_reference(space, py_obj, w_obj) -# return w_obj - -#@cpython_api([PyObject], lltype.Void, header=None) -#def bytearray_dealloc(space, py_obj): -# """Frees allocated PyByteArrayObject resources. -# """ -# py_ba = rffi.cast(PyByteArrayObject, py_obj) -# if py_ba.c_ob_bytes: -# lltype.free(py_ba.c_ob_bytes, flavor="raw") -# from pypy.module.cpyext.object import PyObject_dealloc -# PyObject_dealloc(space, py_obj) - #_______________________________________________________________________ @cpython_api([PyObject], PyObject, result_is_ll=True) diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -11,7 +11,7 @@ from pypy.objspace.std.bytesobject import W_BytesObject ## -## Implementation of PyStringObject +## Implementation of PyBytesObject ## ================================ ## ## The problem @@ -29,15 +29,15 @@ ## Solution ## -------- ## -## PyStringObject contains two additional members: the ob_size and a pointer to a +## PyBytesObject contains two additional members: the ob_size and a pointer to a ## char ob_sval; it may be NULL. ## -## - A string allocated by pypy will be converted into a PyStringObject with a +## - A string allocated by pypy will be converted into a PyBytesObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is ## allocated (with flavor='raw') and content is copied. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyStringObject structure, and a buffer with the specified +## allocate a PyBytesObject structure, and a buffer with the specified ## size+1, but the reference won't be stored in the global map; there is no ## corresponding object in pypy. When from_ref() or Py_INCREF() is called, ## the pypy string is created, and added to the global map of tracked @@ -55,58 +55,58 @@ ## corresponds to the pypy gc-managed string. ## -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyVarObjectFields + \ +PyBytesObjectStruct = lltype.ForwardReference() +PyBytesObject = lltype.Ptr(PyBytesObjectStruct) +PyBytesObjectFields = PyVarObjectFields + \ (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +cpython_struct("PyStringObject", PyBytesObjectFields, PyBytesObjectStruct) @bootstrap_function -def init_stringobject(space): - "Type description of PyStringObject" +def init_bytesobject(space): + "Type description of PyBytesObject" make_typedescr(space.w_str.layout.typedef, - basestruct=PyStringObject.TO, - attach=string_attach, - dealloc=string_dealloc, - realize=string_realize) + basestruct=PyBytesObject.TO, + attach=bytes_attach, + dealloc=bytes_dealloc, + realize=bytes_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocate a PyStringObject and its ob_sval, but without a corresponding - interpreter object. The ob_sval may be mutated, until string_realize() is + Allocate a PyBytesObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until bytes_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) py_obj = typedescr.allocate(space, space.w_str, length) - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str -def string_attach(space, py_obj, w_obj): +def bytes_attach(space, py_obj, w_obj): """ - Copy RPython string object contents to a PyStringObject. The + Copy RPython string object contents to a PyBytesObject. The c_ob_sval must not be modified. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = space.str_w(w_obj) if py_str.c_ob_size < len(s): raise oefmt(space.w_ValueError, - "string_attach called on object with ob_size %d but trying to store %d", - py_str.c_ob_size, len(s)) + "bytes_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL -def string_realize(space, py_obj): +def bytes_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject ob_sval must not + Creates the string in the interpreter. The PyBytesObject ob_sval must not be modified after this call. """ - py_str = rffi.cast(PyStringObject, py_obj) + py_str = rffi.cast(PyBytesObject, py_obj) s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(W_BytesObject, w_type) @@ -117,11 +117,11 @@ return w_obj @cpython_api([PyObject], lltype.Void, header=None) -def string_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. +def bytes_dealloc(space, py_obj): + """Frees allocated PyBytesObject resources. """ - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @@ -154,10 +154,10 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - ref_str = rffi.cast(PyStringObject, ref) + ref_str = rffi.cast(PyBytesObject, ref) if not pyobj_has_w_obj(ref): # XXX Force the ref? - string_realize(space, ref) + bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) @@ -166,7 +166,7 @@ # if no w_str is associated with this ref, # return the c-level ptr as RW if not pyobj_has_w_obj(ref): - py_str = rffi.cast(PyStringObject, ref) + py_str = rffi.cast(PyBytesObject, ref) return py_str.c_ob_sval return _PyString_AsString(space, ref) @@ -183,8 +183,8 @@ from_ref(space, ref)) if not pyobj_has_w_obj(ref): # force the ref - string_realize(space, ref) - ref_str = rffi.cast(PyStringObject, ref) + bytes_realize(space, ref) + ref_str = rffi.cast(PyBytesObject, ref) data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size @@ -200,7 +200,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: - ref = rffi.cast(PyStringObject, ref) + ref = rffi.cast(PyBytesObject, ref) return ref.c_ob_size else: w_obj = from_ref(space, ref) @@ -222,7 +222,7 @@ if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") - py_str = rffi.cast(PyStringObject, ref[0]) + py_str = rffi.cast(PyBytesObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -46,8 +46,8 @@ Py_DecRef(space, py_code) Py_DecRef(space, py_frame.c_f_globals) Py_DecRef(space, py_frame.c_f_locals) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def frame_realize(space, py_obj): """ diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -60,8 +60,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) def code_attach(space, py_obj, w_obj): py_code = rffi.cast(PyCodeObject, py_obj) @@ -80,8 +80,8 @@ py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) Py_DecRef(space, py_code.c_co_filename) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -8,9 +8,12 @@ #endif typedef struct { - PyObject_HEAD - Py_ssize_t ob_size; - PyObject **ob_item; /* XXX optimize to ob_item[] */ + PyObject_VAR_HEAD + PyObject *ob_item[1]; + /* ob_item contains space for 'ob_size' elements. + * Items must normally not be NULL, except during construction when + * the tuple is not yet visible outside the function that builds it. + */ } PyTupleObject; /* defined in varargswrapper.c */ diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -55,8 +55,8 @@ py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Py_DecRef(space, py_func.c_m_module) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -54,6 +54,9 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): + return _dealloc(space, obj) + +def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we # assert that it is really zero here. assert obj.c_ob_refcnt == 0 diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -164,7 +164,7 @@ pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) if pytype.c_tp_itemsize != 0: - itemcount = space.len_w(w_obj) # PyStringObject and subclasses + itemcount = space.len_w(w_obj) # PyBytesObject and subclasses else: itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -46,5 +46,5 @@ py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -38,14 +38,14 @@ @cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): - """Frees allocated PyStringObject resources. + """Frees allocated PyBytesObject resources. """ py_slice = rffi.cast(PySliceObject, py_obj) Py_DecRef(space, py_slice.c_start) Py_DecRef(space, py_slice.c_stop) Py_DecRef(space, py_slice.c_step) - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @@ -73,7 +73,7 @@ length length, and store the length of the slice in slicelength. Out of bounds indices are clipped in a manner consistent with the handling of normal slices. - + Returns 0 on success and -1 on error with exception set.""" if not PySlice_Check(space, w_slice): PyErr_BadInternalCall(space) @@ -88,11 +88,11 @@ """Retrieve the start, stop and step indices from the slice object slice, assuming a sequence of length length. Treats indices greater than length as errors. - + Returns 0 on success and -1 on error with no exception set (unless one of the indices was not None and failed to be converted to an integer, in which case -1 is returned with an exception set). - + You probably do not want to use this function. If you want to use slice objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1858,6 +1858,56 @@ } } +static PyObject* +array_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int ii, nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int ii, nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2117,7 +2167,7 @@ 0, /* tp_setattr */ 0, /* tp_compare */ (reprfunc)array_repr, /* tp_repr */ - 0, /* tp_as_number*/ + &array_as_number, /* tp_as_number*/ &array_as_sequence, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ @@ -2126,7 +2176,8 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -1,6 +1,20 @@ #include "Python.h" #include "structmember.h" +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong + #define PyThing_FromStringAndSize PyUnicode_FromStringAndSize + #define PyThing_FromString PyUnicode_FromString + #define PyThing_Check PyUnicode_Check + #define _PyThing_AsString _PyUnicode_AsString +#else + #define PyThing_FromStringAndSize PyString_FromStringAndSize + #define PyThing_FromString PyString_FromString + #define PyThing_Check PyString_Check + #define _PyThing_AsString PyString_AsString +#endif + typedef struct { PyObject_HEAD int foo; /* the context holder */ @@ -88,7 +102,7 @@ static PyObject * foo_get_name(PyObject *self, void *closure) { - return PyString_FromStringAndSize("Foo Example", 11); + return PyThing_FromStringAndSize("Foo Example", 11); } static PyObject * @@ -114,7 +128,7 @@ { PyObject *format; - format = PyString_FromString(""); + format = PyThing_FromString(""); if (format == NULL) return NULL; return format; } @@ -130,11 +144,11 @@ foo_setattro(fooobject *self, PyObject *name, PyObject *value) { char *name_str; - if (!PyString_Check(name)) { + if (!PyThing_Check(name)) { PyErr_SetObject(PyExc_AttributeError, name); return -1; } - name_str = PyString_AsString(name); + name_str = _PyThing_AsString(name); if (strcmp(name_str, "set_foo") == 0) { long v = PyInt_AsLong(value); @@ -620,20 +634,65 @@ (destructor)custom_dealloc, /*tp_dealloc*/ }; +static PyTypeObject TupleLike = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.TupleLike", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; + + static PyObject *size_of_instances(PyObject *self, PyObject *t) { return PyInt_FromLong(((PyTypeObject *)t)->tp_basicsize); } + +static PyObject * is_TupleLike(PyObject *self, PyObject * t) +{ + int tf = t->ob_type == &TupleLike; + if (t->ob_type->tp_itemsize == 0) + return PyInt_FromLong(-1); + return PyInt_FromLong(tf); +} + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, {"newCustom", (PyCFunction)newCustom, METH_NOARGS, NULL}, {"size_of_instances", (PyCFunction)size_of_instances, METH_O, NULL}, + {"is_TupleLike", (PyCFunction)is_TupleLike, METH_O, NULL}, {NULL, NULL} /* Sentinel */ }; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "foo", + "Module Doc", + -1, + foo_functions, + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_foo(void) + +#else + +#define INITERROR return /* Initialize this module. */ #ifdef __GNUC__ @@ -644,8 +703,16 @@ PyMODINIT_FUNC initfoo(void) +#endif { - PyObject *m, *d; + PyObject *d; +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("foo", foo_functions); +#endif + if (module == NULL) + INITERROR; footype.tp_new = PyType_GenericNew; @@ -654,52 +721,59 @@ MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype) < 0) - return; + INITERROR; if (PyType_Ready(&UnicodeSubtype2) < 0) - return; + INITERROR; if (PyType_Ready(&MetaType) < 0) - return; + INITERROR; if (PyType_Ready(&InitErrType) < 0) - return; + INITERROR; if (PyType_Ready(&SimplePropertyType) < 0) - return; + INITERROR; SimplePropertyType.tp_new = PyType_GenericNew; InitErrType.tp_new = PyType_GenericNew; CustomType.ob_type = &MetaType; if (PyType_Ready(&CustomType) < 0) - return; + INITERROR; UnicodeSubtype3.tp_flags = Py_TPFLAGS_DEFAULT; UnicodeSubtype3.tp_base = &UnicodeSubtype; UnicodeSubtype3.tp_bases = Py_BuildValue("(OO)", &UnicodeSubtype, &CustomType); if (PyType_Ready(&UnicodeSubtype3) < 0) - return; + INITERROR; - m = Py_InitModule("foo", foo_functions); - if (m == NULL) - return; - d = PyModule_GetDict(m); + TupleLike.tp_base = &PyTuple_Type; + if (PyType_Ready(&TupleLike) < 0) + INITERROR; + + + d = PyModule_GetDict(module); if (d == NULL) - return; + INITERROR; if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype3", (PyObject *) &UnicodeSubtype3) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "InitErrType", (PyObject *) &InitErrType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Property", (PyObject *) &SimplePropertyType) < 0) - return; + INITERROR; if (PyDict_SetItemString(d, "Custom", (PyObject *) &CustomType) < 0) - return; + INITERROR; + if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) + INITERROR; +#if PY_MAJOR_VERSION >=3 + return module; +#endif } diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -77,3 +77,10 @@ #assert s == "carray\n_reconstruct\np0\n(S'i'\np1\n(lp2\nI1\naI2\naI3\naI4\natp3\nRp4\n." rra = pickle.loads(s) # rra is arr backwards #assert arr.tolist() == rra.tolist() + + def test_binop_mul_impl(self): + # check that rmul is called + module = self.import_module(name='array') + arr = module.array('i', [2]) + res = [1, 2, 3] * arr + assert res == [1, 2, 3, 1, 2, 3] diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -28,10 +28,10 @@ """ return PyBool_FromLong(PyByteArray_Check(PyTuple_GetItem(args, 0))); """)], prologue='#include ') - assert module.get_hello1() == 'Hello world' - assert module.get_hello2() == 'Hello world\x00' + assert module.get_hello1() == b'Hello world' + assert module.get_hello2() == b'Hello world\x00' assert module.test_Size() - assert module.test_is_bytearray(bytearray("")) + assert module.test_is_bytearray(bytearray(b"")) assert not module.test_is_bytearray(()) def test_bytearray_buffer_init(self): @@ -63,7 +63,7 @@ ]) s = module.getbytearray() assert len(s) == 4 - assert s == 'ab\x00c' + assert s == b'ab\x00c' def test_bytearray_mutable(self): module = self.import_extension('foo', [ @@ -79,9 +79,9 @@ """), ]) s = module.mutable() - if s == '\x00' * 10: + if s == b'\x00' * 10: assert False, "no RW access to bytearray" - assert s[:6] == 'works\x00' + assert s[:6] == b'works\x00' def test_AsByteArray(self): module = self.import_extension('foo', [ @@ -97,18 +97,18 @@ """), ]) s = module.getbytearray() - assert s == 'test' + assert s == b'test' def test_manipulations(self): import sys module = self.import_extension('foo', [ - ("bytearray_from_string", "METH_VARARGS", + ("bytearray_from_bytes", "METH_VARARGS", ''' - return PyByteArray_FromStringAndSize(PyString_AsString( + return PyByteArray_FromStringAndSize(PyBytes_AsString( PyTuple_GetItem(args, 0)), 4); ''' ), - ("str_from_bytearray", "METH_VARARGS", + ("bytes_from_bytearray", "METH_VARARGS", ''' char * buf; int n; @@ -121,7 +121,7 @@ return NULL; } n = PyByteArray_Size(obj); - return PyString_FromStringAndSize(buf, n); + return PyBytes_FromStringAndSize(buf, n); ''' ), ("concat", "METH_VARARGS", @@ -129,7 +129,7 @@ PyObject * ret, *right, *left; PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyUnicode_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -141,16 +141,16 @@ ret = PyByteArray_Concat(ba1, ba2); return ret; """)]) - assert module.bytearray_from_string("huheduwe") == "huhe" - assert module.str_from_bytearray(bytearray('abc')) == 'abc' + assert module.bytearray_from_bytes(b"huheduwe") == b"huhe" + assert module.bytes_from_bytearray(bytearray(b'abc')) == b'abc' if '__pypy__' in sys.builtin_module_names: # CPython only makes an assert. - raises(ValueError, module.str_from_bytearray, 4.0) - ret = module.concat('abc', 'def') - assert ret == 'abcdef' + raises(ValueError, module.bytes_from_bytearray, 4.0) + ret = module.concat(b'abc', b'def') + assert ret == b'abcdef' assert not isinstance(ret, str) assert isinstance(ret, bytearray) - raises(TypeError, module.concat, 'abc', u'def') + raises(TypeError, module.concat, b'abc', u'def') def test_bytearray_resize(self): module = self.import_extension('foo', [ @@ -159,7 +159,7 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyUnicode_FromString("parse failed"); } ba = PyByteArray_FromObject(obj); @@ -168,7 +168,7 @@ oldsize = PyByteArray_Size(ba); if (oldsize == 0) { - return PyString_FromString("oldsize is 0"); + return PyUnicode_FromString("oldsize is 0"); } ret = PyByteArray_Resize(ba, newsize); if (ret != 0) @@ -179,10 +179,9 @@ return ba; ''' )]) - ret = module.bytearray_resize('abc', 6) + ret = module.bytearray_resize(b'abc', 6) assert len(ret) == 6,"%s, len=%d" % (ret, len(ret)) - assert ret == 'abc\x00\x00\x00' - ret = module.bytearray_resize('abcdefghi', 4) + assert ret == b'abc\x00\x00\x00' + ret = module.bytearray_resize(b'abcdefghi', 4) assert len(ret) == 4,"%s, len=%d" % (ret, len(ret)) - assert ret == 'abcd' - + assert ret == b'abcd' diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.bytesobject import new_empty_str, PyStringObject +from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP, generic_cpy_call from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr @@ -10,22 +10,22 @@ import py import sys -class AppTestStringObject(AppTestCpythonExtensionBase): - def test_stringobject(self): +class AppTestBytesObject(AppTestCpythonExtensionBase): + def test_bytesobject(self): module = self.import_extension('foo', [ ("get_hello1", "METH_NOARGS", """ - return PyString_FromStringAndSize( + return PyBytes_FromStringAndSize( "Hello world", 11); """), ("get_hello2", "METH_NOARGS", """ - return PyString_FromString("Hello world"); + return PyBytes_FromString("Hello world"); """), ("test_Size", "METH_NOARGS", """ - PyObject* s = PyString_FromString("Hello world"); - int result = PyString_Size(s); + PyObject* s = PyBytes_FromString("Hello world"); + int result = PyBytes_Size(s); Py_DECREF(s); return PyLong_FromLong(result); @@ -33,38 +33,38 @@ ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - PyString_Size(f); + PyBytes_Size(f); Py_DECREF(f); return NULL; """), - ("test_is_string", "METH_VARARGS", + ("test_is_bytes", "METH_VARARGS", """ - return PyBool_FromLong(PyString_Check(PyTuple_GetItem(args, 0))); + return PyBool_FromLong(PyBytes_Check(PyTuple_GetItem(args, 0))); """)], prologue='#include ') - assert module.get_hello1() == 'Hello world' - assert module.get_hello2() == 'Hello world' + assert module.get_hello1() == b'Hello world' + assert module.get_hello2() == b'Hello world' assert module.test_Size() == 11 raises(TypeError, module.test_Size_exception) - assert module.test_is_string("") - assert not module.test_is_string(()) + assert module.test_is_bytes(b"") + assert not module.test_is_bytes(()) - def test_string_buffer_init(self): + def test_bytes_buffer_init(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ PyObject *s, *t; char* c; - s = PyString_FromStringAndSize(NULL, 4); + s = PyBytes_FromStringAndSize(NULL, 4); if (s == NULL) return NULL; - t = PyString_FromStringAndSize(NULL, 3); + t = PyBytes_FromStringAndSize(NULL, 3); if (t == NULL) return NULL; Py_DECREF(t); - c = PyString_AS_STRING(s); + c = PyBytes_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -72,62 +72,62 @@ return s; """), ]) - s = module.getstring() + s = module.getbytes() assert len(s) == 4 - assert s == 'ab\x00c' + assert s == b'ab\x00c' - def test_string_tp_alloc(self): + def test_bytes_tp_alloc(self): module = self.import_extension('foo', [ ("tpalloc", "METH_NOARGS", """ PyObject *base; PyTypeObject * type; - PyStringObject *obj; - base = PyString_FromString("test"); - if (PyString_GET_SIZE(base) != 4) - return PyLong_FromLong(-PyString_GET_SIZE(base)); + PyBytesObject *obj; + base = PyBytes_FromString("test"); + if (PyBytes_GET_SIZE(base) != 4) + return PyLong_FromLong(-PyBytes_GET_SIZE(base)); type = base->ob_type; if (type->tp_itemsize != 1) return PyLong_FromLong(type->tp_itemsize); - obj = (PyStringObject*)type->tp_alloc(type, 10); - if (PyString_GET_SIZE(obj) != 10) - return PyLong_FromLong(PyString_GET_SIZE(obj)); - /* cannot work, there is only RO access */ - memcpy(PyString_AS_STRING(obj), "works", 6); + obj = (PyBytesObject*)type->tp_alloc(type, 10); + if (PyBytes_GET_SIZE(obj) != 10) + return PyLong_FromLong(PyBytes_GET_SIZE(obj)); + /* cannot work, there is only RO access + memcpy(PyBytes_AS_STRING(obj), "works", 6); */ Py_INCREF(obj); return (PyObject*)obj; """), ('alloc_rw', "METH_NOARGS", ''' - PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); - memcpy(PyString_AS_STRING(obj), "works", 6); + PyObject *obj = _PyObject_NewVar(&PyBytes_Type, 10); + memcpy(PyBytes_AS_STRING(obj), "works", 6); return (PyObject*)obj; '''), ]) s = module.alloc_rw() - assert s == 'works' + '\x00' * 5 + assert s == b'works' + b'\x00' * 5 s = module.tpalloc() - assert s == 'works' + '\x00' * 5 + assert s == b'\x00' * 10 def test_AsString(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyString_FromStringAndSize("test", 4); - char* c = PyString_AsString(s1); - PyObject* s2 = PyString_FromStringAndSize(c, 4); + PyObject* s1 = PyBytes_FromStringAndSize("test", 4); + char* c = PyBytes_AsString(s1); + PyObject* s2 = PyBytes_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), ]) - s = module.getstring() - assert s == 'test' + s = module.getbytes() + assert s == b'test' def test_manipulations(self): module = self.import_extension('foo', [ - ("string_as_string", "METH_VARARGS", + ("bytes_as_string", "METH_VARARGS", ''' - return PyString_FromStringAndSize(PyString_AsString( + return PyBytes_FromStringAndSize(PyBytes_AsString( PyTuple_GetItem(args, 0)), 4); ''' ), @@ -137,22 +137,22 @@ PyObject * left = PyTuple_GetItem(args, 0); Py_INCREF(left); /* the reference will be stolen! */ v = &left; - PyString_Concat(v, PyTuple_GetItem(args, 1)); + PyBytes_Concat(v, PyTuple_GetItem(args, 1)); return *v; """)]) - assert module.string_as_string("huheduwe") == "huhe" - ret = module.concat('abc', 'def') - assert ret == 'abcdef' + assert module.bytes_as_string(b"huheduwe") == b"huhe" + ret = module.concat(b'abc', b'def') + assert ret == b'abcdef' ret = module.concat('abc', u'def') assert not isinstance(ret, str) assert isinstance(ret, unicode) assert ret == 'abcdef' - def test_py_string_as_string_None(self): + def test_py_bytes_as_string_None(self): module = self.import_extension('foo', [ ("string_None", "METH_VARARGS", ''' - if (PyString_AsString(Py_None)) { + if (PyBytes_AsString(Py_None)) { Py_RETURN_NONE; } return NULL; @@ -162,18 +162,18 @@ def test_AsStringAndSize(self): module = self.import_extension('foo', [ - ("getstring", "METH_NOARGS", + ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyString_FromStringAndSize("te\\0st", 5); + PyObject* s1 = PyBytes_FromStringAndSize("te\\0st", 5); char *buf; Py_ssize_t len; - if (PyString_AsStringAndSize(s1, &buf, &len) < 0) + if (PyBytes_AsStringAndSize(s1, &buf, &len) < 0) return NULL; if (len != 5) { PyErr_SetString(PyExc_AssertionError, "Bad Length"); return NULL; } - if (PyString_AsStringAndSize(s1, &buf, NULL) >= 0) { + if (PyBytes_AsStringAndSize(s1, &buf, NULL) >= 0) { PyErr_SetString(PyExc_AssertionError, "Should Have failed"); return NULL; } @@ -183,7 +183,7 @@ return Py_None; """), ]) - module.getstring() + module.getbytes() def test_py_string_as_string_Unicode(self): module = self.import_extension('foo', [ @@ -266,7 +266,7 @@ PyObject *s = args; Py_INCREF(s); PyString_InternInPlace(&s); - if (((PyStringObject*)s)->ob_sstate == SSTATE_NOT_INTERNED) + if (((PyBytesObject*)s)->ob_sstate == SSTATE_NOT_INTERNED) { Py_DECREF(s); s = PyString_FromString("interned error"); @@ -284,7 +284,7 @@ ("test_macro_invocations", "METH_NOARGS", """ PyObject* o = PyString_FromString(""); - PyStringObject* u = (PyStringObject*)o; + PyBytesObject* u = (PyBytesObject*)o; PyString_GET_SIZE(u); PyString_GET_SIZE(o); @@ -301,17 +301,17 @@ ("test_hash", "METH_VARARGS", ''' PyObject* obj = (PyTuple_GetItem(args, 0)); - long hash = ((PyStringObject*)obj)->ob_shash; + long hash = ((PyBytesObject*)obj)->ob_shash; return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - /*int sstate = ((PyStringObject*)s)->ob_sstate; + /*int sstate = ((PyBytesObject*)s)->ob_sstate; printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - /*sstate = ((PyStringObject*)s)->ob_sstate; + /*sstate = ((PyBytesObject*)s)->ob_sstate; printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); @@ -345,7 +345,7 @@ PyObject_HEAD_INIT(NULL) 0, /* ob_size */ "bar.string_", /* tp_name*/ - sizeof(PyStringObject), /* tp_basicsize*/ + sizeof(PyBytesObject), /* tp_basicsize*/ 0 /* tp_itemsize */ }; @@ -401,7 +401,7 @@ return NULL; } destptr = PyString_AS_STRING(obj); - ((PyStringObject *)obj)->ob_shash = -1; + ((PyBytesObject *)obj)->ob_shash = -1; memcpy(destptr, data, itemsize); return obj; } @@ -420,8 +420,8 @@ assert type(a).__name__ == 'string_' assert a == 'abc' -class TestString(BaseApiTest): - def test_string_resize(self, space, api): +class TestBytes(BaseApiTest): + def test_bytes_resize(self, space, api): py_str = new_empty_str(space, 10) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') py_str.c_ob_sval[0] = 'a' @@ -429,14 +429,14 @@ py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 3) - py_str = rffi.cast(PyStringObject, ar[0]) + py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 3 assert py_str.c_ob_sval[1] == 'b' assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 10) - py_str = rffi.cast(PyStringObject, ar[0]) + py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 10 assert py_str.c_ob_sval[1] == 'b' assert py_str.c_ob_sval[10] == '\x00' diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -51,7 +51,7 @@ api._PyTuple_Resize(ar, 10) assert api.PyTuple_Size(ar[0]) == 10 for i in range(3, 10): - rffi.cast(PyTupleObject, py_tuple).c_ob_item[i] = make_ref( + rffi.cast(PyTupleObject, ar[0]).c_ob_item[i] = make_ref( space, space.wrap(42 + i)) w_tuple = from_ref(space, ar[0]) assert space.int_w(space.len(w_tuple)) == 10 @@ -151,3 +151,8 @@ """), ]) module.run() + + def test_tuple_subclass(self): + module = self.import_module(name='foo') + a = module.TupleLike([1, 2, 3]) + assert module.is_TupleLike(a) == 1 diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -2,10 +2,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.debug import fatalerror_notb from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, + build_type_checkers, PyVarObjectFields, cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - make_ref, from_ref, decref, + make_ref, from_ref, decref, incref, pyobj_has_w_obj, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -29,8 +29,8 @@ PyTupleObjectStruct = lltype.ForwardReference() PyTupleObject = lltype.Ptr(PyTupleObjectStruct) ObjectItems = rffi.CArray(PyObject) -PyTupleObjectFields = PyObjectFields + \ - (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +PyTupleObjectFields = PyVarObjectFields + \ + (("ob_item", ObjectItems),) cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) @bootstrap_function @@ -56,14 +56,12 @@ tuple_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_tuple.layout.typedef) - py_obj = typedescr.allocate(space, space.w_tuple) + py_obj = typedescr.allocate(space, space.w_tuple, length) py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_ob_item = lltype.malloc(ObjectItems, length, - flavor='raw', zero=True, - add_memory_pressure=True) - py_tup.c_ob_size = length - return py_tup + p = py_tup.c_ob_item + for i in range(py_tup.c_ob_size): + p[i] = lltype.nullptr(PyObject.TO) + return py_obj def tuple_attach(space, py_obj, w_obj): """ @@ -71,23 +69,24 @@ buffer must not be modified. """ items_w = space.fixedview(w_obj) - l = len(items_w) - p = lltype.malloc(ObjectItems, l, flavor='raw', - add_memory_pressure=True) + py_tup = rffi.cast(PyTupleObject, py_obj) + length = len(items_w) + if py_tup.c_ob_size < length: + raise oefmt(space.w_ValueError, + "tuple_attach called on object with ob_size %d but trying to store %d", + py_tup.c_ob_size, length) i = 0 try: - while i < l: - p[i] = make_ref(space, items_w[i]) + while i < length: + py_tup.c_ob_item[i] = make_ref(space, items_w[i]) i += 1 except: while i > 0: i -= 1 - decref(space, p[i]) - lltype.free(p, flavor='raw') + ob = py_tup.c_ob_item[i] + py_tup.c_ob_item[i] = lltype.nullptr(PyObject.TO) + decref(space, ob) raise - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_ob_size = l - py_tup.c_ob_item = p def tuple_realize(space, py_obj): """ @@ -108,7 +107,9 @@ "converting a PyTupleObject into a W_TupleObject, " "but found NULLs as items") items_w[i] = w_item - w_obj = space.newtuple(items_w) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_TupleObject, w_type) + w_obj.__init__(items_w) track_reference(space, py_obj, w_obj) return w_obj @@ -118,18 +119,16 @@ """ py_tup = rffi.cast(PyTupleObject, py_obj) p = py_tup.c_ob_item - if p: - for i in range(py_tup.c_ob_size): - decref(space, p[i]) - lltype.free(p, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + for i in range(py_tup.c_ob_size): + decref(space, p[i]) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) #_______________________________________________________________________ @cpython_api([Py_ssize_t], PyObject, result_is_ll=True) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return new_empty_tuple(space, size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PyTuple_SetItem(space, ref, index, py_obj): @@ -185,25 +184,25 @@ ref = p_ref[0] if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - ref = rffi.cast(PyTupleObject, ref) - oldsize = ref.c_ob_size - oldp = ref.c_ob_item - newp = lltype.malloc(ObjectItems, newsize, zero=True, flavor='raw', - add_memory_pressure=True) + oldref = rffi.cast(PyTupleObject, ref) + oldsize = oldref.c_ob_size + p_ref[0] = new_empty_tuple(space, newsize) + newref = rffi.cast(PyTupleObject, p_ref[0]) try: if oldsize < newsize: to_cp = oldsize else: to_cp = newsize for i in range(to_cp): - newp[i] = oldp[i] + ob = oldref.c_ob_item[i] + incref(space, ob) + newref.c_ob_item[i] = ob except: - lltype.free(newp, flavor='raw') + decref(space, p_ref[0]) + p_ref[0] = lltype.nullptr(PyObject.TO) raise - ref.c_ob_item = newp - ref.c_ob_size = newsize - lltype.free(oldp, flavor='raw') - # in this version, p_ref[0] never needs to be updated + finally: + decref(space, ref) return 0 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -167,7 +167,7 @@ py_memberdescr.c_d_member = w_obj.member def memberdescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? member = rffi.cast(lltype.Ptr(PyMemberDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_MemberDescr, w_type) @@ -192,7 +192,7 @@ py_methoddescr.c_d_method = w_obj.ml def classmethoddescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? method = rffi.cast(lltype.Ptr(PyMethodDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_PyCClassMethodObject, w_type) @@ -201,7 +201,7 @@ return w_obj def methoddescr_realize(space, obj): - # XXX NOT TESTED When is this ever called? + # XXX NOT TESTED When is this ever called? method = rffi.cast(lltype.Ptr(PyMethodDef), obj) w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(W_PyCMethodObject, w_type) @@ -371,6 +371,8 @@ # (minimally, if tp_basicsize is zero we copy it from the base) if not pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize + if pto.c_tp_itemsize < base_pto.c_tp_itemsize: + pto.c_tp_itemsize = base_pto.c_tp_itemsize flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) @@ -564,7 +566,7 @@ if space.is_w(w_type, space.w_str): # Special case: str doesn't support get_raw_address(), so we have a # custom get*buffer that instead gives the address of the char* in the - # PyStringObject*! + # PyBytesObject*! c_buf.c_bf_getreadbuffer = llhelper( str_getreadbuffer.api_func.functype, str_getreadbuffer.api_func.get_wrapper(space)) @@ -597,7 +599,7 @@ @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): - from pypy.module.cpyext.object import PyObject_dealloc + from pypy.module.cpyext.object import _dealloc obj_pto = rffi.cast(PyTypeObjectPtr, obj) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) @@ -608,7 +610,7 @@ heaptype = rffi.cast(PyHeapTypeObject, obj) Py_DecRef(space, heaptype.c_ht_name) Py_DecRef(space, base_pyo) - PyObject_dealloc(space, obj) + _dealloc(space, obj) def type_alloc(space, w_metatype, itemsize=0): @@ -659,6 +661,8 @@ subtype_dealloc.api_func.get_wrapper(space)) if space.is_w(w_type, space.w_str): pto.c_tp_itemsize = 1 + elif space.is_w(w_type, space.w_tuple): + pto.c_tp_itemsize = rffi.sizeof(PyObject) # buffer protocol setup_buffer_procs(space, w_type, pto) @@ -778,7 +782,7 @@ if py_type.c_ob_type: w_metatype = from_ref(space, rffi.cast(PyObject, py_type.c_ob_type)) - else: + else: # Somehow the tp_base type is created with no ob_type, notably # PyString_Type and PyBaseString_Type # While this is a hack, cpython does it as well. @@ -793,10 +797,10 @@ # inheriting tp_as_* slots base = py_type.c_tp_base if base: - if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number - if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence - if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping - if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer + if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number + if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence + if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping + if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer return w_obj diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -90,8 +90,9 @@ Py_DecRef(space, py_unicode.c_defenc) if py_unicode.c_str: lltype.free(py_unicode.c_str, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) + + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): diff --git a/rpython/doc/getting-started.rst b/rpython/doc/getting-started.rst --- a/rpython/doc/getting-started.rst +++ b/rpython/doc/getting-started.rst @@ -142,8 +142,8 @@ Translating Full Programs ~~~~~~~~~~~~~~~~~~~~~~~~~ -To translate full RPython programs, there is the script ``translate.py`` in -:source:`rpython/translator/goal`. Examples for this are a slightly changed version of +To translate full RPython programs, there is the script ``bin/rpython`` in +:source:`rpython/bin/`. Examples for this are a slightly changed version of Pystone:: python bin/rpython translator/goal/targetrpystonedalone diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -1,6 +1,7 @@ from rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from rpython.memory.gctransform.support import (get_rtti, - _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor) + _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor, + ll_report_finalizer_error) from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.flowspace.model import Constant from rpython.rtyper.lltypesystem.lloperation import llop @@ -58,6 +59,9 @@ self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() + self.finalizer_triggers = [] + self.finalizer_queue_indexes = {} # {fq: index} + def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): # XXX same behavior for zero=True: in theory that's wrong if TYPE._is_atomic(): @@ -115,6 +119,39 @@ self.finalizer_funcptrs[TYPE] = fptr return fptr + def get_finalizer_queue_index(self, hop): + fq_tag = hop.spaceop.args[0].value + assert 'FinalizerQueue TAG' in fq_tag.expr + fq = fq_tag.default + try: + index = self.finalizer_queue_indexes[fq] + except KeyError: + index = len(self.finalizer_queue_indexes) + assert index == len(self.finalizer_triggers) + # + def ll_finalizer_trigger(): + try: + fq.finalizer_trigger() + except Exception as e: + ll_report_finalizer_error(e) + ll_trigger = self.annotate_finalizer(ll_finalizer_trigger, [], + lltype.Void) + self.finalizer_triggers.append(ll_trigger) + self.finalizer_queue_indexes[fq] = index + return index + + def gct_gc_fq_register(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + v_ptr = hop.spaceop.args[1] + hop.genop("boehm_fq_register", [c_index, v_ptr]) + + def gct_gc_fq_next_dead(self, hop): + index = self.get_finalizer_queue_index(hop) + c_index = rmodel.inputconst(lltype.Signed, index) + hop.genop("boehm_fq_next_dead", [c_index], + resultvar = hop.spaceop.result) + def gct_weakref_create(self, hop): v_instance, = hop.spaceop.args v_addr = hop.genop("cast_ptr_to_adr", [v_instance], diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -166,7 +166,19 @@ }) if '__int128_t' in rffi.TYPES: - _ctypes_cache[rffi.__INT128_T] = ctypes.c_longlong # XXX: Not right at all. But for some reason, It started by while doing JIT compile after a merge with default. Can't extend ctypes, because thats a python standard, right? + class c_int128(ctypes.Array): # based on 2 ulongs + _type_ = ctypes.c_uint64 + _length_ = 2 + @property + def value(self): + if sys.byteorder == 'little': + res = self[0] | (self[1] << 64) + else: + res = self[1] | (self[0] << 64) + if res >= (1 << 127): + res -= 1 << 128 + return res + _ctypes_cache[rffi.__INT128_T] = c_int128 # for unicode strings, do not use ctypes.c_wchar because ctypes # automatically converts arrays into unicode strings. diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -825,3 +825,13 @@ assert charp2str(p2) == "helLD" free_charp(p1) free_charp(p2) + +def test_sign_when_casting_uint_to_larger_int(): + from rpython.rtyper.lltypesystem import rffi + from rpython.rlib.rarithmetic import r_uint32, r_uint64 + # + value = 0xAAAABBBB + assert cast(lltype.SignedLongLong, r_uint32(value)) == value + if hasattr(rffi, '__INT128_T'): + value = 0xAAAABBBBCCCCDDDD + assert cast(rffi.__INT128_T, r_uint64(value)) == value diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py --- a/rpython/translator/backendopt/finalizer.py +++ b/rpython/translator/backendopt/finalizer.py @@ -36,8 +36,8 @@ raise FinalizerError(msg) else: result = self.analyze_direct_call(graph) - if result is self.top_result(): - log.red('old-style non-light finalizer: %r' % (graph,)) + #if result is self.top_result(): + # log.red('old-style non-light finalizer: %r' % (graph,)) return result def analyze_simple_operation(self, op, graphinfo): diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -212,6 +212,24 @@ compile_extra=['-DPYPY_USING_BOEHM_GC'], )) + gct = self.db.gctransformer + gct.finalizer_triggers = tuple(gct.finalizer_triggers) # stop changing + sourcelines = [''] + for trig in gct.finalizer_triggers: + sourcelines.append('RPY_EXTERN void %s(void);' % ( + self.db.get(trig),)) + sourcelines.append('') + sourcelines.append('void (*boehm_fq_trigger[])(void) = {') + for trig in gct.finalizer_triggers: + sourcelines.append('\t%s,' % (self.db.get(trig),)) + sourcelines.append('\tNULL') + sourcelines.append('};') + sourcelines.append('struct boehm_fq_s *boehm_fq_queues[%d];' % ( + len(gct.finalizer_triggers),)) + sourcelines.append('') + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=['\n'.join(sourcelines)])) + return eci def gc_startup_code(self): diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -121,7 +121,7 @@ #define OP_LLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \ r = (long long)(((unsigned long long)(x)) << (y)) #define OP_LLLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, 128); \ - r = (__int128)(((unsigned __int128)(x)) << (y)) + r = (__int128_t)(((__uint128_t)(x)) << (y)) #define OP_UINT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \ r = (x) << (y) @@ -157,7 +157,7 @@ #define OP_CAST_UINT_TO_INT(x,r) r = (Signed)(x) #define OP_CAST_INT_TO_UINT(x,r) r = (Unsigned)(x) #define OP_CAST_INT_TO_LONGLONG(x,r) r = (long long)(x) -#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128)(x) +#define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128_t)(x) #define OP_CAST_CHAR_TO_INT(x,r) r = (Signed)((unsigned char)(x)) #define OP_CAST_INT_TO_CHAR(x,r) r = (char)(x) #define OP_CAST_PTR_TO_INT(x,r) r = (Signed)(x) /* XXX */ diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -73,9 +73,17 @@ #ifdef PYPY_USING_BOEHM_GC +struct boehm_fq_s { + void *obj; + struct boehm_fq_s *next; +}; +RPY_EXTERN void (*boehm_fq_trigger[])(void); + int boehm_gc_finalizer_lock = 0; void boehm_gc_finalizer_notifier(void) { + int i; + boehm_gc_finalizer_lock++; while (GC_should_invoke_finalizers()) { if (boehm_gc_finalizer_lock > 1) { @@ -86,6 +94,11 @@ } GC_invoke_finalizers(); } + + i = 0; + while (boehm_fq_trigger[i]) + boehm_fq_trigger[i++](); + boehm_gc_finalizer_lock--; } @@ -100,6 +113,28 @@ GC_finalize_on_demand = 1; GC_set_warn_proc(mem_boehm_ignore); } + +void boehm_fq_callback(void *obj, void *rawfqueue) +{ + struct boehm_fq_s **fqueue = rawfqueue; + struct boehm_fq_s *node = GC_malloc(sizeof(void *) * 2); + if (!node) + return; /* ouch, too bad */ + node->obj = obj; + node->next = *fqueue; + *fqueue = node; +} + +void *boehm_fq_next_dead(struct boehm_fq_s **fqueue) +{ + struct boehm_fq_s *node = *fqueue; + if (node != NULL) { + *fqueue = node->next; + return node->obj; + } + else + return NULL; +} #endif /* BOEHM GC */ diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -104,13 +104,21 @@ RPY_EXTERN int boehm_gc_finalizer_lock; RPY_EXTERN void boehm_gc_startup_code(void); RPY_EXTERN void boehm_gc_finalizer_notifier(void); +struct boehm_fq_s; +RPY_EXTERN struct boehm_fq_s *boehm_fq_queues[]; +RPY_EXTERN void (*boehm_fq_trigger[])(void); +RPY_EXTERN void boehm_fq_callback(void *, void *); +RPY_EXTERN void *boehm_fq_next_dead(struct boehm_fq_s **); #define OP_GC__DISABLE_FINALIZERS(r) boehm_gc_finalizer_lock++ #define OP_GC__ENABLE_FINALIZERS(r) (boehm_gc_finalizer_lock--, \ boehm_gc_finalizer_notifier()) -#define OP_GC_FQ_REGISTER(tag, obj, r) /* ignored so far */ -#define OP_GC_FQ_NEXT_DEAD(tag, r) (r = NULL) +#define OP_BOEHM_FQ_REGISTER(tagindex, obj, r) \ + GC_REGISTER_FINALIZER(obj, boehm_fq_callback, \ + boehm_fq_queues + tagindex, NULL, NULL) +#define OP_BOEHM_FQ_NEXT_DEAD(tagindex, r) \ + r = boehm_fq_next_dead(boehm_fq_queues + tagindex) #endif /* PYPY_USING_BOEHM_GC */ diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -393,20 +393,36 @@ assert res[3] == compute_hash(d) assert res[4] == compute_hash(("Hi", None, (7.5, 2, d))) - def test_finalizer_queue_is_at_least_ignored(self): + def test_finalizer_queue(self): class A(object): - pass + def __init__(self, i): + self.i = i + class Glob: + triggered = 0 + glob = Glob() class FQ(rgc.FinalizerQueue): Class = A + triggered = 0 def finalizer_trigger(self): - debug.debug_print("hello!") # not called so far + glob.triggered += 1 fq = FQ() # def fn(): - fq.register_finalizer(A()) + for i in range(1000): + fq.register_finalizer(A(i)) rgc.collect() rgc.collect() - fq.next_dead() + if glob.triggered == 0: + print "not triggered!" + return 50 + seen = {} + for i in range(1000): + a = fq.next_dead() + assert a.i not in seen + seen[a.i] = True + if len(seen) < 500: + print "seen only %d!" % len(seen) + return 51 return 42 f = self.getcompiled(fn) From pypy.commits at gmail.com Tue Jun 28 15:54:35 2016 From: pypy.commits at gmail.com (mattip) Date: Tue, 28 Jun 2016 12:54:35 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: a rough draft of some slides for the compiler workshop Message-ID: <5772d5fb.67c0c20a.fc0e0.0ef1@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5642:2532bf0a23bd Date: 2016-06-28 22:53 +0300 http://bitbucket.org/pypy/extradoc/changeset/2532bf0a23bd/ Log: a rough draft of some slides for the compiler workshop diff --git a/talk/compiler-workshop-2016/pypy.rst b/talk/compiler-workshop-2016/pypy.rst new file mode 100644 --- /dev/null +++ b/talk/compiler-workshop-2016/pypy.rst @@ -0,0 +1,29 @@ +PyPy - DRAFT +============ + +A quick overview of PyPy for other Python-compiler developers + + +What is Our Niche +----------------- + +PyPy is a mature production-ready framework automatically speeding up pure +python code by factors of 2-5 in commercial settings. + +What We Bring to the Table +-------------------------- + +Our core is based on cutting-edge JIT compiler +research, written in Python. Being written in Python means we can quickly +try out (and reject or incorporate) new ideas. For instance, our STM work has +uncovered subtle bugs in CLANG and GCC, or the fact that we can easily try out +new register allocation strategies. + +What We Need Help With +---------------------- + +We seem to be more successful in improving core technologies than creating a +end-user friendly nicely packaged distribution. We are also extremely +under-funded. For instance, we could be doing alot more for data science but +are moving slowly forward on a volunteer basis with C-API compatibility. + From pypy.commits at gmail.com Wed Jun 29 02:19:13 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 23:19:13 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix the comment and replace again DEFAULT with a singleton, as it is not Message-ID: <57736861.e409c20a.91be4.ffff9ba6@mx.google.com> Author: Armin Rigo Branch: Changeset: r85441:8a0060eb072f Date: 2016-06-29 08:18 +0200 http://bitbucket.org/pypy/pypy/changeset/8a0060eb072f/ Log: Fix the comment and replace again DEFAULT with a singleton, as it is not RPython any more diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -293,7 +293,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) From pypy.commits at gmail.com Wed Jun 29 02:19:15 2016 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Jun 2016 23:19:15 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <57736863.472e1c0a.56f27.ffff8210@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85442:1e8833dfdd80 Date: 2016-06-29 08:20 +0200 http://bitbucket.org/pypy/pypy/changeset/1e8833dfdd80/ Log: in-progress diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -813,9 +813,11 @@ def fget_f_builtins(self, space): return self.get_builtin().getdict(space) + def get_f_back(self): + return ExecutionContext.getnextframe_nohidden(self) + def fget_f_back(self, space): - f_back = ExecutionContext.getnextframe_nohidden(self) - return self.space.wrap(f_back) + return self.space.wrap(self.get_f_back()) def fget_f_lasti(self, space): return self.space.wrap(self.last_instr) diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -1,7 +1,9 @@ import sys from rpython.rlib import revdb from rpython.rlib.debug import make_sure_not_resized -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter import gateway, typedef class DBState: @@ -17,8 +19,8 @@ #make_sure_not_resized(dbstate.breakpoint_funcnames) #make_sure_not_resized(dbstate.watch_progs) make_sure_not_resized(dbstate.metavars) - #revdb.register_debug_command(revdb.CMD_PRINT, lambda_print) - #revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace) + revdb.register_debug_command(revdb.CMD_PRINT, lambda_print) + revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace) #revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals) #revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints) #revdb.register_debug_command(revdb.CMD_MOREINFO, lambda_moreinfo) @@ -39,3 +41,103 @@ "'$%d' refers to an object created later in time", oparg) return w_var + +def fetch_cur_frame(): + ec = dbstate.space.getexecutioncontext() + frame = ec.topframeref() + if frame is None: + revdb.send_output("No stack.\n") + return frame + +def compile(source, mode): + space = dbstate.space + compiler = space.createcompiler() + code = compiler.compile(source, '', mode, 0, + hidden_applevel=True) + return code + + +class W_RevDBOutput(W_Root): + softspace = 0 + + def __init__(self, space): + self.space = space + + def descr_write(self, w_buffer): + space = self.space + if space.isinstance_w(w_buffer, space.w_unicode): + w_buffer = space.call_method(w_buffer, 'encode', + space.wrap('utf-8')) # safe? + revdb.send_output(space.str_w(w_buffer)) + +W_RevDBOutput.typedef = typedef.TypeDef( + "revdb_output", + write = gateway.interp2app(W_RevDBOutput.descr_write), + softspace = typedef.interp_attrproperty("softspace", W_RevDBOutput), + ) + + +def command_print(cmd, expression): + frame = fetch_cur_frame() + if frame is None: + return + space = dbstate.space + try: + code = compile(expression, 'exec') + w_revdb_output = space.wrap(W_RevDBOutput(space)) + space.sys.setdictvalue(space, 'stdout', w_revdb_output) + space.sys.setdictvalue(space, 'stderr', w_revdb_output) + try: + code.exec_code(space, + frame.get_w_globals(), + frame.getdictscope()) + + except OperationError as operationerr: + w_type = operationerr.w_type + w_value = operationerr.get_w_value(space) + w_traceback = space.wrap(operationerr.get_traceback()) + + # set the sys.last_xxx attributes + space.setitem(space.sys.w_dict, space.wrap('last_type'), w_type) + space.setitem(space.sys.w_dict, space.wrap('last_value'), w_value) + space.setitem(space.sys.w_dict, space.wrap('last_traceback'), + w_traceback) + + # call sys.excepthook if present + w_hook = space.sys.getdictvalue(space, 'excepthook') + if w_hook is None: + raise + space.call_function(w_hook, w_type, w_value, w_traceback) + return + + except OperationError as e: + revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) + return +lambda_print = lambda: command_print + + +def show_frame(frame, indent=''): + code = frame.getcode() + lineno = frame.get_last_lineno() + revdb.send_output('%sFile "%s", line %d in %s\n%s ' % ( + indent, code.co_filename, lineno, code.co_name, indent)) + revdb.send_linecache(code.co_filename, lineno) + +def command_backtrace(cmd, extra): + frame = fetch_cur_frame() + if frame is None: + return + if cmd.c_arg1 == 0: + show_frame(frame) + else: + revdb.send_output("Traceback (most recent call last):\n") + frames = [] + while frame is not None: + frames.append(frame) + if len(frames) == 200: + revdb.send_output(" ...\n") + break + frame = frame.get_f_back() + while len(frames) > 0: + show_frame(frames.pop(), indent=' ') +lambda_backtrace = lambda: command_backtrace diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -18,6 +18,7 @@ CMD_ATTACHID = 6 CMD_CHECKWATCH = 7 CMD_WATCHVALUES = 8 +ANSWER_LINECACHE= 19 ANSWER_TEXT = 20 ANSWER_MOREINFO = 21 ANSWER_NEXTNID = 22 @@ -49,6 +50,9 @@ def send_watch(text, ok_flag): send_answer(ANSWER_WATCH, ok_flag, extra=text) +def send_linecache(filename, linenum): + send_answer(ANSWER_LINECACHE, linenum, extra=filename) + def current_time(): """For RPython debug commands: returns the current time.""" return llop.revdb_get_value(lltype.SignedLongLong, 'c') diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -47,6 +47,10 @@ # if breakpoint_mode=='i': ignored, never sent ANSWER_BREAKPOINT = -24 +# print one line of a file to the console, for CMD_PRINT +# Message(ANSWER_LINECACHE, linenum, extra=filename) +ANSWER_LINECACHE = 19 + # print text to the console, for CMD_PRINT and others # Message(ANSWER_TEXT, extra=text) ANSWER_TEXT = 20 diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -1,4 +1,5 @@ import sys, os, struct, socket, errno, subprocess +import linecache from rpython.translator.revdb import ancillary from rpython.translator.revdb.message import * @@ -170,6 +171,12 @@ elif msg.cmd == ANSWER_READY: self.update_times(msg) break + elif msg.cmd == ANSWER_LINECACHE: + line = linecache.getline(msg.extra, msg.arg1) + if not line.endswith('\n'): + line += '\n' + sys.stdout.write(line) + sys.stdout.flush() elif msg.cmd == ANSWER_NEXTNID and pgroup is not None: uid = msg.arg1 if uid < pgroup.initial_uid: @@ -507,6 +514,8 @@ def show_backtrace(self, complete=1): """Show the backtrace. """ + if complete: + self.active.tainted = True self.active.send(Message(CMD_BACKTRACE, complete)) self.active.print_text_answer() From pypy.commits at gmail.com Wed Jun 29 04:11:39 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 01:11:39 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <577382bb.4a2e1c0a.895d0.fffffa09@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85443:2564a81ce049 Date: 2016-06-29 10:12 +0200 http://bitbucket.org/pypy/pypy/changeset/2564a81ce049/ Log: in-progress diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -1,6 +1,8 @@ import sys from rpython.rlib import revdb from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.objectmodel import specialize +from rpython.rtyper.annlowlevel import cast_gcref_to_instance from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter import gateway, typedef @@ -8,40 +10,63 @@ class DBState: extend_syntax_with_dollar_num = False + breakpoint_funcnames = [] + printed_objects = {} metavars = [] + watch_progs = [] + watch_futures = {} dbstate = DBState() def setup_revdb(space): + """Called at run-time, before the space is set up. + + The various register_debug_command() lines attach functions + to some commands that 'revdb.py' can call, if we are running + in replay mode. + """ assert space.config.translation.reverse_debugger dbstate.space = space - #make_sure_not_resized(dbstate.breakpoint_funcnames) - #make_sure_not_resized(dbstate.watch_progs) + dbstate.w_future = space.w_Ellipsis # a random prebuilt object + + make_sure_not_resized(dbstate.breakpoint_funcnames) + make_sure_not_resized(dbstate.watch_progs) make_sure_not_resized(dbstate.metavars) + revdb.register_debug_command(revdb.CMD_PRINT, lambda_print) revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace) #revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals) #revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints) #revdb.register_debug_command(revdb.CMD_MOREINFO, lambda_moreinfo) - #revdb.register_debug_command("ALLOCATING", lambda_allocating) - #revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid) + revdb.register_debug_command("ALLOCATING", lambda_allocating) + revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid) #revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch) #revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) -def load_metavar(oparg): + +def load_metavar(index): + assert index >= 0 space = dbstate.space metavars = dbstate.metavars - w_var = metavars[oparg] if oparg < len(metavars) else None + w_var = metavars[index] if index < len(metavars) else None if w_var is None: raise oefmt(space.w_NameError, "no constant object '$%d'", - oparg) - if w_var is space.w_Ellipsis: + index) + if w_var is dbstate.w_future: raise oefmt(space.w_RuntimeError, "'$%d' refers to an object created later in time", - oparg) + index) return w_var +def set_metavar(index, w_obj): + assert index >= 0 + if index >= len(dbstate.metavars): + missing = index + 1 - len(dbstate.metavars) + dbstate.metavars = dbstate.metavars + [None] * missing + dbstate.metavars[index] = w_obj + + def fetch_cur_frame(): ec = dbstate.space.getexecutioncontext() frame = ec.topframeref() @@ -76,6 +101,25 @@ softspace = typedef.interp_attrproperty("softspace", W_RevDBOutput), ) +def revdb_displayhook(space, w_obj): + """Modified sys.displayhook() that also outputs '$NUM = ', + for non-prebuilt objects. Such objects are then recorded in + 'printed_objects'. + """ + if space.is_w(w_obj, space.w_None): + return + uid = revdb.get_unique_id(w_obj) + if uid > 0: + dbstate.printed_objects[uid] = w_obj + revdb.send_nextnid(uid) # outputs '$NUM = ' + space.setitem(space.builtin.w_dict, space.wrap('_'), w_obj) + revdb.send_output(space.str_w(space.repr(w_obj))) + revdb.send_output("\n") + + at specialize.memo() +def get_revdb_displayhook(space): + return space.wrap(gateway.interp2app(revdb_displayhook)) + def command_print(cmd, expression): frame = fetch_cur_frame() @@ -85,8 +129,10 @@ try: code = compile(expression, 'exec') w_revdb_output = space.wrap(W_RevDBOutput(space)) + w_displayhook = get_revdb_displayhook(space) space.sys.setdictvalue(space, 'stdout', w_revdb_output) space.sys.setdictvalue(space, 'stderr', w_revdb_output) + space.sys.setdictvalue(space, 'displayhook', w_displayhook) try: code.exec_code(space, frame.get_w_globals(), @@ -141,3 +187,29 @@ while len(frames) > 0: show_frame(frames.pop(), indent=' ') lambda_backtrace = lambda: command_backtrace + + +def command_allocating(uid, gcref): + w_obj = cast_gcref_to_instance(W_Root, gcref) + dbstate.printed_objects[uid] = w_obj + try: + index_metavar = dbstate.watch_futures.pop(uid) + except KeyError: + pass + else: + set_metavar(index_metavar, w_obj) +lambda_allocating = lambda: command_allocating + + +def command_attachid(cmd, extra): + space = dbstate.space + index_metavar = cmd.c_arg1 + uid = cmd.c_arg2 + try: + w_obj = dbstate.printed_objects[uid] + except KeyError: + # uid not found, probably a future object + dbstate.watch_futures[uid] = index_metavar + w_obj = dbstate.w_future + set_metavar(index_metavar, w_obj) +lambda_attachid = lambda: command_attachid From pypy.commits at gmail.com Wed Jun 29 05:49:08 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 02:49:08 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Prebuilt weakrefs Message-ID: <57739994.22ecc20a.3cc8a.fffff5ff@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85444:172c97e74bff Date: 2016-06-29 11:50 +0200 http://bitbucket.org/pypy/pypy/changeset/172c97e74bff/ Log: Prebuilt weakrefs diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -389,6 +389,8 @@ it with WEAKREF_AFTERWARDS_ALIVE. */ if (!RPY_RDB_REPLAY) r->re_off_prev = recording_offset(); + else + r->re_off_prev = 1; /* any number > 0 */ RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); @@ -420,29 +422,34 @@ struct WEAKLINK *r = (struct WEAKLINK *)weakref; void *result = r->re_addr; if (result && flag_io_disabled == FID_REGULAR_MODE) { - char alive; - if (!RPY_RDB_REPLAY) { - if (r->re_off_prev <= 0) { - fprintf(stderr, "bug in weakrefs: bad previous offset %lld\n", - (long long)r->re_off_prev); - exit(1); + if (r->re_off_prev < 0) { + fprintf(stderr, "bug in weakrefs: bad previous offset %lld\n", + (long long)r->re_off_prev); + exit(1); + } + if (r->re_off_prev == 0) { + /* A prebuilt weakref. Don't record anything */ + } + else { + char alive; + if (!RPY_RDB_REPLAY) { + patch_prev_offset(r->re_off_prev, WEAKREF_AFTERWARDS_DEAD, + WEAKREF_AFTERWARDS_ALIVE); + r->re_off_prev = recording_offset(); } - patch_prev_offset(r->re_off_prev, WEAKREF_AFTERWARDS_DEAD, - WEAKREF_AFTERWARDS_ALIVE); - r->re_off_prev = recording_offset(); - } - RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); + RPY_REVDB_EMIT(alive = WEAKREF_AFTERWARDS_DEAD;, char _e, alive); - if (RPY_RDB_REPLAY) { - switch (alive) { - case WEAKREF_AFTERWARDS_DEAD: - r->re_addr = NULL; - break; - case WEAKREF_AFTERWARDS_ALIVE: - break; - default: - fprintf(stderr, "bad weakref_deref byte in log\n"); - exit(1); + if (RPY_RDB_REPLAY) { + switch (alive) { + case WEAKREF_AFTERWARDS_DEAD: + r->re_addr = NULL; + break; + case WEAKREF_AFTERWARDS_ALIVE: + break; + default: + fprintf(stderr, "bad weakref_deref byte in log\n"); + exit(1); + } } } } diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -151,6 +151,22 @@ x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() + def test_prebuilt_weakref(self): + class X: + pass + x1 = X() + x1.foobar = 9 + wr = weakref.ref(x1) + def main(argv): + X().foobar = 43 + return wr().foobar + self.compile(main, backendopt=False) + out = self.run('Xx') + rdb = self.fetch_rdb([self.exename, 'Xx']) + # the weakref is prebuilt, so doesn't generate any WEAKREF_xxx + x = rdb.next('q'); assert x == 0 # number of stop points + assert rdb.done() + def test_finalizer_light_ignored(self): py.test.skip("lightweight finalizers could be skipped, but that " "requires also skipping (instead of recording) any " From pypy.commits at gmail.com Wed Jun 29 05:55:33 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 02:55:33 -0700 (PDT) Subject: [pypy-commit] pypy default: Oops, this test would segfault if Boehm didn't manage to free all 1000 Message-ID: <57739b15.e85dc20a.e0213.fffff80a@mx.google.com> Author: Armin Rigo Branch: Changeset: r85445:c1e9b92562a8 Date: 2016-06-29 11:56 +0200 http://bitbucket.org/pypy/pypy/changeset/c1e9b92562a8/ Log: Oops, this test would segfault if Boehm didn't manage to free all 1000 dead objects diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -416,8 +416,10 @@ print "not triggered!" return 50 seen = {} - for i in range(1000): + while True: a = fq.next_dead() + if a is None: + break assert a.i not in seen seen[a.i] = True if len(seen) < 500: From pypy.commits at gmail.com Wed Jun 29 06:10:03 2016 From: pypy.commits at gmail.com (Raemi) Date: Wed, 29 Jun 2016 03:10:03 -0700 (PDT) Subject: [pypy-commit] stmgc default: add possible todo Message-ID: <57739e7b.8438c20a.8a4bc.0181@mx.google.com> Author: Remi Meier Branch: Changeset: r1990:514c0fe60b9b Date: 2016-06-29 12:09 +0200 http://bitbucket.org/pypy/stmgc/changeset/514c0fe60b9b/ Log: add possible todo diff --git a/c8/TODO b/c8/TODO --- a/c8/TODO +++ b/c8/TODO @@ -1,3 +1,12 @@ + +- investigate if userfaultfd() helps: + http://kernelnewbies.org/Linux_4.3#head-3deefea7b0add8c1b171b0e72ce3b69c5ed35cb0 + + AFAICS, we could avoid the current in-kernel-pagefault+SIGSEGV-handler for + making pages accessible on-demand, and replace that mechanism with a + user-pagefault "handler". That should save on the number of needed VMAs and + possibly be faster (although I'm quite unsure of that). + ################## Issue: lee_router_tm in one of its versions uses a temporary grid (big list) From pypy.commits at gmail.com Wed Jun 29 06:14:37 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 29 Jun 2016 03:14:37 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: some additions to the compiler workshop in austin Message-ID: <57739f8d.4aa6c20a.20045.048a@mx.google.com> Author: Richard Plangger Branch: extradoc Changeset: r5643:dda61f9232d2 Date: 2016-06-29 12:14 +0200 http://bitbucket.org/pypy/extradoc/changeset/dda61f9232d2/ Log: some additions to the compiler workshop in austin diff --git a/talk/compiler-workshop-2016/pypy.rst b/talk/compiler-workshop-2016/pypy.rst --- a/talk/compiler-workshop-2016/pypy.rst +++ b/talk/compiler-workshop-2016/pypy.rst @@ -3,13 +3,15 @@ A quick overview of PyPy for other Python-compiler developers - What is Our Niche ----------------- PyPy is a mature production-ready framework automatically speeding up pure python code by factors of 2-5 in commercial settings. +The PyPy community offers several tools to inspect and optimize Python programs. +Examples: vmprof, cffi + What We Bring to the Table -------------------------- @@ -19,6 +21,21 @@ uncovered subtle bugs in CLANG and GCC, or the fact that we can easily try out new register allocation strategies. +Our JIT is not only advanced, but runs on all major platforms (Linux, Windows, MacOS, ...) including +four different CPU architectures (x86, arm, ppc, s390x). + +We have built a high level language to aid the construction of VMs (called *RPython*) +It imports the complete program +Flow graphs -> Annotation -> RTyping -> Code generation + +Advantages: +* Whole program optimizations (take that C) +* Deliver features fast, without sacrificing speed +* Loosely coupled (JIT, Interp., RPython) + +Downsides: +* Takes a while to translate + What We Need Help With ---------------------- @@ -27,3 +44,29 @@ under-funded. For instance, we could be doing alot more for data science but are moving slowly forward on a volunteer basis with C-API compatibility. +Our interests lie in still providing the confort of the Python eco system, +but not sacrificing execution time. Some details (e.g. garbage collection scheme) +have some impact on user programs. We could use a lot more help in identifying and resolving +some of these issues. If programs do not run out of the box, most users will stick to CPython +because their performance problems are not that of an issue (at that point in time). +If we could resolve those issues (funded, or externally contributed) we would create a much +better user experience. + +We are also working on Micro NumPy, which provides the kernels for numerical operations. +It is very much complete, but still some features are missing. We would love to have a +partner/company that would help us to complete NumPy for PyPy. + +We are open to changes to our JIT scheme. We are working on both high level optimizations and +backend oriented changes. Ideas would be to mitigate some issues with our tracing JIT compiler +(or even build a region based compiler) and many more. Most of these aspects are covered quite well by +our core development team, but we will eventually take another big step in near future towards the 7th version +of our JIT. + +Other Interesting Aspects +------------------------- + +(Backup slides) + +Some words about how our JIT compiler works? + +We sprint 2-3 times a year (open to the public). From pypy.commits at gmail.com Wed Jun 29 06:21:53 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 29 Jun 2016 03:21:53 -0700 (PDT) Subject: [pypy-commit] pypy default: enhance the documentation about sets in rpython (related to my question on IRC) Message-ID: <5773a141.03c31c0a.b017f.3a5e@mx.google.com> Author: Richard Plangger Branch: Changeset: r85446:dea2cb868b71 Date: 2016-06-29 12:18 +0200 http://bitbucket.org/pypy/pypy/changeset/dea2cb868b71/ Log: enhance the documentation about sets in rpython (related to my question on IRC) diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -135,6 +135,12 @@ hash functions and custom equality will not be honored. Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions. +**sets** + + sets are not directly supported in RPython. Instead you should use a + plain dict and fill the values with None. Values in that dict + will not consume space. + **list comprehensions** May be used to create allocated, initialized arrays. From pypy.commits at gmail.com Wed Jun 29 06:43:09 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 03:43:09 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <5773a63d.8a40c20a.b3956.0f63@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85447:155739abc604 Date: 2016-06-29 12:44 +0200 http://bitbucket.org/pypy/pypy/changeset/155739abc604/ Log: in-progress diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -152,6 +152,9 @@ Like bytecode_trace() but doesn't invoke any other events besides the trace function. """ + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import potential_stop_point + potential_stop_point(frame) if (frame.get_w_f_trace() is None or self.is_tracing or self.gettrace() is None): return diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -1,11 +1,11 @@ import sys from rpython.rlib import revdb from rpython.rlib.debug import make_sure_not_resized -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.annlowlevel import cast_gcref_to_instance from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import gateway, typedef +from pypy.interpreter import gateway, typedef, pycode class DBState: @@ -45,6 +45,90 @@ #revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) +pycode.PyCode.co_revdb_linestarts = None # or a string: a list of bits + + +def potential_stop_point(frame): + if not we_are_translated(): + return + # + # We only record a stop_point at every line, not every bytecode. + # Uses roughly the same algo as ExecutionContext.run_trace_func() + # to know where the line starts are, but tweaked for speed, + # avoiding the quadratic complexity when run N times with a large + # code object. A potential difference is that we only record + # where the line starts are; the "We jumped backwards in the same + # line" case of run_trace_func() is not fully reproduced. + # + code = frame.pycode + lstart = code.co_revdb_linestarts + if lstart is None: + lstart = build_co_revdb_linestarts(code) + index = frame.last_instr + c = lstart[index >> 3] + if ord(c) & (1 << (index & 7)): + stop_point_at_start_of_line() + +def build_co_revdb_linestarts(code): + # inspired by findlinestarts() in the 'dis' standard module + assert len(code.co_code) > 0 + bits = [False] * len(code.co_code) + bits[0] = True + lnotab = code.co_lnotab + addr = 0 + p = 0 + newline = True + while p + 1 < len(lnotab): + byte_incr = ord(lnotab[p]) + line_incr = ord(lnotab[p+1]) + if byte_incr: + if newline: + if addr < len(bits): + bits[addr] = True + newline = False + addr += byte_incr + if line_incr: + newline = True + p += 2 + if newline: + if addr < len(bits): + bits[addr] = True + # + byte_list = [] + pending = 0 + nextval = 1 + for bit_is_set in bits: + if bit_is_set: + pending |= nextval + if nextval < 128: + nextval <<= 1 + else: + byte_list.append(chr(pending)) + pending = 0 + nextval = 1 + if nextval != 1: + byte_list.append(chr(pending)) + lstart = ''.join(byte_list) + code.co_revdb_linestarts = lstart + return lstart + + +def stop_point_at_start_of_line(): + if revdb.watch_save_state(): + any_watch_point = False + #for prog, watch_id, expected in dbstate.watch_progs: + # any_watch_point = True + # got = _watch_expr(prog) + # if got != expected: + # break + #else: + watch_id = -1 + revdb.watch_restore_state(any_watch_point) + if watch_id != -1: + revdb.breakpoint(watch_id) + revdb.stop_point() + + def load_metavar(index): assert index >= 0 space = dbstate.space @@ -113,7 +197,11 @@ dbstate.printed_objects[uid] = w_obj revdb.send_nextnid(uid) # outputs '$NUM = ' space.setitem(space.builtin.w_dict, space.wrap('_'), w_obj) - revdb.send_output(space.str_w(space.repr(w_obj))) + # do str_w(repr()) only now: if w_obj was produced successfully, + # but its repr crashes because it tries to do I/O, then we already + # have it recorded in '_' and in '$NUM ='. + s = space.str_w(space.repr(w_obj)) + revdb.send_output(s) revdb.send_output("\n") @specialize.memo() diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/test_reverse_debugging.py @@ -0,0 +1,28 @@ +import dis +from pypy.interpreter.reverse_debugging import * + + +class FakeCode: + def __init__(self, co_code, co_lnotab): + self.co_firstlineno = 43 + self.co_code = co_code + self.co_lnotab = co_lnotab + self.co_revdb_linestarts = None + + +def test_build_co_revdb_linestarts(): + fake_lnotab = ("\x01\x02\x03\x04" + "\x00\xFF\x20\x30\x00\xFF\x00\x40" + "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00") + code = FakeCode("?" * sum(map(ord, fake_lnotab[0::2])), fake_lnotab) + lstart = build_co_revdb_linestarts(code) + assert lstart is code.co_revdb_linestarts + + expected_starts = set() + for addr, lineno in dis.findlinestarts(code): + expected_starts.add(addr) + + for index in range(len(code.co_code)): + c = lstart[index >> 3] + found = ord(c) & (1 << (index & 7)) + assert (found != 0) == (index in expected_starts) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -91,15 +91,15 @@ def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""): msg = self.recv() - assert msg.cmd == cmd + assert msg.cmd == cmd, msg if arg1 is not Ellipsis: - assert msg.arg1 == arg1 + assert msg.arg1 == arg1, msg if arg2 is not Ellipsis: - assert msg.arg2 == arg2 + assert msg.arg2 == arg2, msg if arg3 is not Ellipsis: - assert msg.arg3 == arg3 + assert msg.arg3 == arg3, msg if extra is not Ellipsis: - assert msg.extra == extra + assert msg.extra == extra, msg return msg def expect_ready(self): @@ -207,6 +207,9 @@ child = ReplayProcess(initial_subproc.pid, s1) msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis) self.total_stop_points = msg.arg2 + if self.total_stop_points == 0: + raise ValueError("%r does not contain any stop point" % + (revdb_log_filename,)) child.expect_ready() self.initial_uid = child.currently_created_objects From pypy.commits at gmail.com Wed Jun 29 07:29:28 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 04:29:28 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: in-progress Message-ID: <5773b118.89acc20a.7fe38.274f@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85448:45db8fd3e7a1 Date: 2016-06-29 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/45db8fd3e7a1/ Log: in-progress diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -45,7 +45,7 @@ #revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) -pycode.PyCode.co_revdb_linestarts = None # or a string: a list of bits +pycode.PyCode.co_revdb_linestarts = None # or a string: an array of bits def potential_stop_point(frame): @@ -71,28 +71,26 @@ def build_co_revdb_linestarts(code): # inspired by findlinestarts() in the 'dis' standard module - assert len(code.co_code) > 0 bits = [False] * len(code.co_code) - bits[0] = True - lnotab = code.co_lnotab - addr = 0 - p = 0 - newline = True - while p + 1 < len(lnotab): - byte_incr = ord(lnotab[p]) - line_incr = ord(lnotab[p+1]) - if byte_incr: - if newline: - if addr < len(bits): - bits[addr] = True - newline = False - addr += byte_incr - if line_incr: - newline = True - p += 2 - if newline: - if addr < len(bits): - bits[addr] = True + if not code.hidden_applevel: + lnotab = code.co_lnotab + addr = 0 + p = 0 + newline = 1 + while p + 1 < len(lnotab): + byte_incr = ord(lnotab[p]) + line_incr = ord(lnotab[p+1]) + if byte_incr: + if newline != 0: + if addr < len(bits): + bits[addr] = True + newline = 0 + addr += byte_incr + newline |= line_incr + p += 2 + if newline: + if addr < len(bits): + bits[addr] = True # byte_list = [] pending = 0 diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py --- a/pypy/interpreter/test/test_reverse_debugging.py +++ b/pypy/interpreter/test/test_reverse_debugging.py @@ -1,5 +1,6 @@ import dis from pypy.interpreter.reverse_debugging import * +from hypothesis import given, strategies, example class FakeCode: @@ -10,11 +11,14 @@ self.co_revdb_linestarts = None -def test_build_co_revdb_linestarts(): - fake_lnotab = ("\x01\x02\x03\x04" - "\x00\xFF\x20\x30\x00\xFF\x00\x40" - "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00") - code = FakeCode("?" * sum(map(ord, fake_lnotab[0::2])), fake_lnotab) + at given(strategies.binary()) + at example("\x01\x02\x03\x04" + "\x00\xFF\x20\x30\x00\xFF\x00\x40" + "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00") +def test_build_co_revdb_linestarts(lnotab): + if len(lnotab) & 1: + lnotab = lnotab + '\x00' # make the length even + code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab) lstart = build_co_revdb_linestarts(code) assert lstart is code.co_revdb_linestarts diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -173,9 +173,9 @@ break elif msg.cmd == ANSWER_LINECACHE: line = linecache.getline(msg.extra, msg.arg1) - if not line.endswith('\n'): - line += '\n' - sys.stdout.write(line) + if line == '': + line = '?' + sys.stdout.write(line.strip() + '\n') sys.stdout.flush() elif msg.cmd == ANSWER_NEXTNID and pgroup is not None: uid = msg.arg1 diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,10 +29,14 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 0 /* enable to print locations to stderr of all the EMITs */ -# define _RPY_REVDB_PRINT(args) fprintf args +#if 1 /* enable to print locations to stderr of all the EMITs */ +# define _RPY_REVDB_PRINT(mode) \ + fprintf(stderr, \ + "%s:%d: %0*llx\n", \ + __FILE__, __LINE__, 2 * sizeof(_e), \ + ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #else -# define _RPY_REVDB_PRINT(args) /* nothing */ +# define _RPY_REVDB_PRINT(mode) /* nothing */ #endif @@ -41,9 +45,7 @@ normal_code \ { \ decl_e = variable; \ - _RPY_REVDB_PRINT((stderr, "%s:%d: write %0*llx\n", \ - __FILE__, __LINE__, \ - 2 * sizeof(_e), (unsigned long long)_e)); \ + _RPY_REVDB_PRINT("write"); \ memcpy(rpy_revdb.buf_p, &_e, sizeof(_e)); \ if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit) \ rpy_reverse_db_flush(); \ @@ -54,9 +56,7 @@ char *_end1 = _src + sizeof(_e); \ memcpy(&_e, _src, sizeof(_e)); \ rpy_revdb.buf_p = _end1; \ - _RPY_REVDB_PRINT((stderr, "%s:%d: read %0*llx\n", \ - __FILE__, __LINE__, \ - 2 * sizeof(_e), (unsigned long long)_e)); \ + _RPY_REVDB_PRINT("read"); \ if (_end1 >= rpy_revdb.buf_limit) \ rpy_reverse_db_fetch(__FILE__, __LINE__); \ variable = _e; \ From pypy.commits at gmail.com Wed Jun 29 09:02:30 2016 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 29 Jun 2016 06:02:30 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: change the code builder to resolve issue for the vector regsiters (float) 32-63, because 0-31 overlap with floating point currently for vector reg. allocation 0-31 are not used. they come in handy to splat floating point values Message-ID: <5773c6e6.0476c20a.f1859.4f57@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85449:946ddc31e87b Date: 2016-06-29 15:01 +0200 http://bitbucket.org/pypy/pypy/changeset/946ddc31e87b/ Log: change the code builder to resolve issue for the vector regsiters (float) 32-63, because 0-31 overlap with floating point currently for vector reg. allocation 0-31 are not used. they come in handy to splat floating point values diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -65,6 +65,7 @@ XX2 = Form("fvrT", "fvrB", "XO6") XX3 = Form("fvrT", "fvrA", "fvrB", "XO9") XX3_2 = Form("fvrT", "fvrA", "fvrB", "OE", "XO11") +XX3_splat = Form("fvrT", "fvrA", "fvrB", "DM", "XO13", "OE") XV = Form("ivrT", "rA", "rB", "XO1") VX = Form("ivrT", "ivrA", "ivrB", "XO8") VC = Form("ivrT", "ivrA", "ivrB", "XO12", "OE") @@ -653,6 +654,14 @@ vcmpequdx = VC(4, XO12=199, OE=1) vcmpequd = VC(4, XO12=199, OE=0) + # permute/splat + # splat low of A, and low of B + xxspltdl = XX3_splat(60, XO13=10, OE=0, DM=0b00) + # splat high of A, and high of B + xxspltdh = XX3_splat(60, XO13=10, OE=0, DM=0b11) + # generic splat + xxspltd = XX3_splat(60, XO13=10, OE=0) + # INTEGER # ------- diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py --- a/rpython/jit/backend/ppc/ppc_field.py +++ b/rpython/jit/backend/ppc/ppc_field.py @@ -44,8 +44,8 @@ "TO": ( 6, 10), "UIMM": (16, 31), "fvrT": (6, 31, 'unsigned', regname._V, 'overlap'), - "fvrA": (11, 31, 'unsigned', regname._V, 'overlap'), - "fvrB": (16, 31, 'unsigned', regname._V, 'overlap'), + "fvrA": (11, 29, 'unsigned', regname._V, 'overlap'), + "fvrB": (16, 30, 'unsigned', regname._V, 'overlap'), # low vector register T (low in a sense: # can only address 32 vector registers) "ivrT": (6, 10, 'unsigned', regname._V), @@ -66,6 +66,8 @@ "XO10": (26, 31), "XO11": (22, 28), "XO12": (22, 31), + "XO13": (24, 28), + "DM": (22, 23), "LL": ( 9, 10), "SIM": (11, 15), } diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py --- a/rpython/jit/backend/ppc/rassemblermaker.py +++ b/rpython/jit/backend/ppc/rassemblermaker.py @@ -50,10 +50,10 @@ body.append('vrT1 = (%s & 31) << 21 | (%s & 32) >> 5' % (value, value)) value = 'vrT1' elif field.name == 'fvrA': - body.append('fvrA1 = ((%s & 31) << 15 | (%s & 32) >> 5) << 2' % (value, value)) + body.append('fvrA1 = (%s & 31) << 14 | (%s & 32) >> 5' % (value, value)) value = 'fvrA1' elif field.name == 'fvrB': - body.append('fvrB1 = ((%s & 31) << 10 | (%s & 32) >> 5) << 1' % (value, value)) + body.append('fvrB1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value)) value = 'fvrB1' if isinstance(field, IField): body.append('v |= ((%3s >> 2) & r_uint(%#05x)) << 2' % (value, field.mask)) diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py --- a/rpython/jit/backend/ppc/register.py +++ b/rpython/jit/backend/ppc/register.py @@ -4,7 +4,7 @@ ALL_REGS = [RegisterLocation(i) for i in range(32)] ALL_FLOAT_REGS = [FPRegisterLocation(i) for i in range(32)] ALL_INTEGER_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(32)] -ALL_FLOAT_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(64)] +ALL_FLOAT_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(32,64)] r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16,\ r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31\ @@ -19,12 +19,12 @@ ivr24, ivr25, ivr26, ivr27, ivr28, ivr29, ivr30, ivr31\ = ALL_FLOAT_REGS -vr0, vr1, vr2, vr3, vr4, vr5, vr6, vr7, vr8, vr9, vr10, vr11, vr12, vr13, \ - vr14, vr15, vr16, vr17, vr18, vr19, vr20, vr21, vr22, vr23, vr24, vr25, \ - vr26, vr27, vr28, vr29, vr30, vr31, vr32, vr33, vr34, vr35, vr36, vr37, \ - vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \ - vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \ - vr61, vr62, vr63 = ALL_FLOAT_VECTOR_REGS +# the first 32 vector register are partly shared with the normal floating point +# registers, since there are so many registers, we just take the upper 31 ones +vr32, vr33, vr34, vr35, vr36, vr37, \ + vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \ + vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \ + vr61, vr62, vr63 = ALL_FLOAT_VECTOR_REGS NONVOLATILES = [r14, r15, r16, r17, r18, r19, r20, r21, r22, r23, diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -479,7 +479,10 @@ self.mc.load_imm(tloc, srcloc.value) self.mc.lxvd2x(res, 0, tloc.value) elif size == 8: - self.mc.vmr(res, srcloc.value, srcloc.value) + # splat the low of src to both slots in res + src = srcloc.value + #import pdb; pdb.set_trace() + self.mc.xxspltdl(res, src, src) else: notimplemented("[ppc/assembler] vec expand in this combination not supported") @@ -804,7 +807,7 @@ l0 = self.expand_float(op.bytesize, arg) res = self.force_allocate_vector_reg(op) else: - l0 = self.ensure_vector_reg(arg) + l0 = self.ensure_reg(arg) res = self.force_allocate_vector_reg(op) return [res, l0] diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -440,7 +440,7 @@ myjitdriver.jit_merge_point() va[i] = va[i] + variable i += 1 - val = va[0] + val = va[d//2] lltype.free(va, flavor='raw') return val res = self.meta_interp(f, [60,58.4547]) From pypy.commits at gmail.com Wed Jun 29 11:01:28 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 08:01:28 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <5773e2c8.4ccf1c0a.bc01c.2253@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85450:62e370af096e Date: 2016-06-29 16:00 +0100 http://bitbucket.org/pypy/pypy/changeset/62e370af096e/ Log: hg merge default diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -421,7 +421,7 @@ if not (space.isinstance_w(w_obj, space.w_str) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, - "AST string must be of type str or unicode") + "AST string must be of type str or unicode") return w_obj def get_field(space, w_node, name, optional): diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py --- a/pypy/interpreter/pyparser/gendfa.py +++ b/pypy/interpreter/pyparser/gendfa.py @@ -294,7 +294,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -164,7 +164,7 @@ pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) if pytype.c_tp_itemsize != 0: - itemcount = space.len_w(w_obj) # PyStringObject and subclasses + itemcount = space.len_w(w_obj) # PyBytesObject and subclasses else: itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -202,7 +202,6 @@ return PyLong_FromLong((long) ((short *)ap->ob_item)[i]); } - static int h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { @@ -2317,6 +2316,56 @@ } } +static PyObject* +array_multiply(PyObject* obj1, PyObject* obj2) +{ + if (PyList_Check(obj1) && ((arrayobject*)obj2)->ob_descr->typecode == 'i' && Py_SIZE(obj2) == 1) + { + int ii, nn; + int n = PyList_Size(obj1); + PyObject *v = getarrayitem(obj2, 0); + long i = PyLong_AsLong(v); // XXX: error checking? + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj1, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if (PyList_Check(obj2) && ((arrayobject*)obj1)->ob_descr->typecode == 'i' && Py_SIZE(obj1) == 1) + { + int ii, nn; + int n = PyList_Size(obj2); + PyObject *v = getarrayitem(obj1, 0); + int i = ((PyIntObject*)v)->ob_ival; + PyObject * ret = PyList_New(n*i); + for (ii = 0; ii < i; ii++) + for (nn = 0; nn < n; nn++) + { + v = PyList_GetItem(obj2, nn); + PyList_SetItem(ret, nn+ii*n, v); + } + return ret; + } + else if(obj1->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + else if(obj2->ob_type == &Arraytype) + fprintf(stderr, "\nCannot multiply array of type %c and %s\n", + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyNumberMethods array_as_number = { + (binaryfunc)NULL, /* nb_add*/ + (binaryfunc)NULL, /* nb_subtract */ + (binaryfunc)array_multiply, /* nb_multiply */ + (binaryfunc)NULL, /* nb_divide */ +}; + static PyMappingMethods array_as_mapping = { (lenfunc)array_length, (binaryfunc)array_subscr, @@ -2586,7 +2635,7 @@ 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)array_repr, /* tp_repr */ - 0, /* tp_as_number*/ + &array_as_number, /* tp_as_number*/ &array_as_sequence, /* tp_as_sequence*/ &array_as_mapping, /* tp_as_mapping*/ 0, /* tp_hash */ @@ -2595,7 +2644,8 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -67,3 +67,10 @@ b'\x02\0\0\0' b'\x03\0\0\0' b'\x04\0\0\0') + + def test_binop_mul_impl(self): + # check that rmul is called + module = self.import_module(name='array') + arr = module.array('i', [2]) + res = [1, 2, 3] * arr + assert res == [1, 2, 3, 1, 2, 3] diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -135,6 +135,12 @@ hash functions and custom equality will not be honored. Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions. +**sets** + + sets are not directly supported in RPython. Instead you should use a + plain dict and fill the values with None. Values in that dict + will not consume space. + **list comprehensions** May be used to create allocated, initialized arrays. diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -416,8 +416,10 @@ print "not triggered!" return 50 seen = {} - for i in range(1000): + while True: a = fq.next_dead() + if a is None: + break assert a.i not in seen seen[a.i] = True if len(seen) < 500: From pypy.commits at gmail.com Wed Jun 29 12:17:27 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 09:17:27 -0700 (PDT) Subject: [pypy-commit] pypy py3k: Create space.newunicode() Message-ID: <5773f497.cdcf1c0a.1df2.69a3@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85452:d6b47988f9f3 Date: 2016-06-29 16:48 +0100 http://bitbucket.org/pypy/pypy/changeset/d6b47988f9f3/ Log: Create space.newunicode() diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -32,7 +32,7 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.tupleobject import W_AbstractTupleObject, W_TupleObject from pypy.objspace.std.typeobject import W_TypeObject, TypeCache -from pypy.objspace.std.unicodeobject import W_UnicodeObject, wrapunicode +from pypy.objspace.std.unicodeobject import W_UnicodeObject class StdObjSpace(ObjSpace): @@ -166,9 +166,9 @@ else: lst.append(unichr(ch)) unicode_x = u''.join(lst) - return wrapunicode(self, unicode_x) + return self.newunicode(unicode_x) if isinstance(x, unicode): - return wrapunicode(self, x) + return self.newunicode(x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, W_Root): @@ -342,6 +342,9 @@ return W_BytesObject(s) wrapbytes = newbytes + def newunicode(self, uni): + return W_UnicodeObject(uni) + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -19,7 +19,7 @@ from pypy.objspace.std.stringmethods import StringMethods from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT -__all__ = ['W_UnicodeObject', 'wrapunicode', 'encode_object', 'decode_object', +__all__ = ['W_UnicodeObject', 'encode_object', 'decode_object', 'unicode_from_object', 'unicode_to_decimal_w'] @@ -465,10 +465,6 @@ return len(prefix) == 0 -def wrapunicode(space, uni): - return W_UnicodeObject(uni) - - def _isidentifier(u): if not u: return False From pypy.commits at gmail.com Wed Jun 29 12:17:25 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 09:17:25 -0700 (PDT) Subject: [pypy-commit] pypy py3k: Rename wrapbytes to newbytes, for consistency with newint, newfloat, etc. Message-ID: <5773f495.571a1c0a.227b4.ffffde27@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85451:f44a76b03795 Date: 2016-06-29 16:34 +0100 http://bitbucket.org/pypy/pypy/changeset/f44a76b03795/ Log: Rename wrapbytes to newbytes, for consistency with newint, newfloat, etc. diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -657,10 +657,6 @@ W_BytesObject.EMPTY = W_BytesObject('') -def wrapstr(space, s): - return W_BytesObject(s) - - def getbytevalue(space, w_value): value = space.getindex_w(w_value, None) if not 0 <= value < 256: diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -16,7 +16,7 @@ # Object imports from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.bytearrayobject import W_BytearrayObject -from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject, wrapstr +from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject from pypy.objspace.std.complexobject import W_ComplexObject from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject from pypy.objspace.std.floatobject import W_FloatObject @@ -129,9 +129,6 @@ assert typedef is not None return self.fromcache(TypeCache).getorbuild(typedef) - def wrapbytes(self, x): - return wrapstr(self, x) - @specialize.argtype(1) def wrap(self, x): "Wraps the Python value 'x' into one of the wrapper classes." @@ -341,6 +338,10 @@ def newbuffer(self, w_obj): return W_MemoryView(w_obj) + def newbytes(self, s): + return W_BytesObject(s) + wrapbytes = newbytes + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) From pypy.commits at gmail.com Wed Jun 29 12:20:19 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 09:20:19 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Starts to work, almost. Message-ID: <5773f543.e655c20a.fc180.ffffaa97@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85453:2070975d17bc Date: 2016-06-29 18:21 +0200 http://bitbucket.org/pypy/pypy/changeset/2070975d17bc/ Log: Starts to work, almost. diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -670,6 +670,12 @@ OP_CAST_ADR_TO_PTR = OP_CAST_POINTER OP_CAST_OPAQUE_PTR = OP_CAST_POINTER + def OP_CAST_PTR_TO_INT(self, op): + if self.db.reverse_debugger: + from rpython.translator.revdb import gencsupp + return gencsupp.cast_ptr_to_int(self, op) + return self.OP_CAST_POINTER(op) + def OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE(self, op): return ('%s = *(long *)(((char *)%s) + sizeof(struct pypy_header0));' ' /* length_of_simple_gcarray_from_opaque */' diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -160,7 +160,6 @@ #define OP_CAST_INT_TO_LONGLONGLONG(x,r) r = (__int128_t)(x) #define OP_CAST_CHAR_TO_INT(x,r) r = (Signed)((unsigned char)(x)) #define OP_CAST_INT_TO_CHAR(x,r) r = (char)(x) -#define OP_CAST_PTR_TO_INT(x,r) r = (Signed)(x) /* XXX */ #define OP_TRUNCATE_LONGLONG_TO_INT(x,r) r = (Signed)(x) #define OP_TRUNCATE_LONGLONGLONG_TO_INT(x,r) r = (Signed)(x) diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -25,6 +25,10 @@ return 'rpy_reverse_db_register_destructor(%s, %s);' % ( funcgen.expr(op.args[0]), funcgen.expr(op.args[1])) +def cast_ptr_to_int(funcgen, op): + return '%s = RPY_REVDB_CAST_PTR_TO_INT(%s);' % ( + funcgen.expr(op.result), funcgen.expr(op.args[0])) + def prepare_database(db): FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR, lltype.Ptr(rstr.STR)], diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1183,6 +1183,9 @@ RPY_EXTERN int rpy_reverse_db_fq_register(void *obj) { + /*fprintf(stderr, "FINALIZER_TREE: %lld -> %p\n", + ((struct pypy_header0 *)obj)->h_uid, obj); + */ if (!RPY_RDB_REPLAY) { return 0; /* recording */ } diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -29,14 +29,19 @@ RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]); RPY_EXTERN void rpy_reverse_db_teardown(void); -#if 1 /* enable to print locations to stderr of all the EMITs */ +#if 0 /* enable to print locations to stderr of all the EMITs */ # define _RPY_REVDB_PRINT(mode) \ fprintf(stderr, \ "%s:%d: %0*llx\n", \ __FILE__, __LINE__, 2 * sizeof(_e), \ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) +# define _RPY_REVDB_PRUID() \ + fprintf(stderr, \ + "%s:%d: obj %llu\n", \ + __FILE__, __LINE__, (unsigned long long) uid) #else # define _RPY_REVDB_PRINT(mode) /* nothing */ +# define _RPY_REVDB_PRUID() /* nothing */ #endif @@ -72,6 +77,7 @@ uid = rpy_reverse_db_unique_id_break(expr); \ rpy_revdb.unique_id_seen = uid + 1; \ ((struct pypy_header0 *)expr)->h_uid = uid; \ + _RPY_REVDB_PRUID(); \ } while (0) #define OP_REVDB_STOP_POINT(r) \ @@ -113,6 +119,12 @@ #define OP_REVDB_CALL_DESTRUCTOR(obj, r) \ rpy_reverse_db_call_destructor(obj) +/* Used only for getting a fast hash value that doesn't change too + often (with the minimark GC, it changes at most once). Here, + we'll just return the UID. */ +#define RPY_REVDB_CAST_PTR_TO_INT(obj) (((struct pypy_header0 *)obj)->h_uid) + + RPY_EXTERN void rpy_reverse_db_flush(void); RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line); RPY_EXTERN void rpy_reverse_db_stop_point(void); From pypy.commits at gmail.com Wed Jun 29 12:22:13 2016 From: pypy.commits at gmail.com (Raemi) Date: Wed, 29 Jun 2016 09:22:13 -0700 (PDT) Subject: [pypy-commit] stmgc default: fix not removing noconflict objs from certain lists Message-ID: <5773f5b5.8999c20a.bfc73.ffffa658@mx.google.com> Author: Remi Meier Branch: Changeset: r1992:ffae8e3eb78b Date: 2016-06-29 13:37 +0200 http://bitbucket.org/pypy/stmgc/changeset/ffae8e3eb78b/ Log: fix not removing noconflict objs from certain lists After resetting a noconflict obj with committed state, the obj should appear to be untouched in the current transaction. This was not the case, as the obj was still in some lists where it shouldn't be. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -151,6 +151,7 @@ } static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj); /* forward */ +static void undo_modifications_to_single_obj(int segment_num, object_t *only_obj); /* forward */ static bool _stm_validate(void) { @@ -268,7 +269,8 @@ backup copy completely: */ /* XXX: this browses through the whole list of modified fragments; this may become a problem... */ - reset_modified_from_backup_copies(my_segnum, obj); + undo_modifications_to_single_obj(my_segnum, obj); + continue; } @@ -1276,6 +1278,37 @@ invoke_general_finalizers(tl); } +static void undo_modifications_to_single_obj(int segment_num, object_t *obj) +{ + /* special function used for noconflict objs to reset all their + * modifications and make them appear untouched in the current transaction. + * I.e., reset modifications and remove from all lists. */ + + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); + + reset_modified_from_backup_copies(segment_num, obj); + + LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/, + { + if (item == obj) { + _reset_object_cards(pseg, item, CARD_CLEAR, false, false); + /* copy last element over this one (HACK) */ + _lst->count -= 1; + _lst->items[_i] = _lst->items[_lst->count]; + break; + } + }); + LIST_FOREACH_R(pseg->objects_pointing_to_nursery, object_t * /*item*/, + { + if (item == obj) { + /* copy last element over this one (HACK) */ + _lst->count -= 1; + _lst->items[_i] = _lst->items[_lst->count]; + break; + } + }); +} + static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj) { #pragma push_macro("STM_PSEGMENT") @@ -1284,6 +1317,9 @@ #undef STM_SEGMENT assert(modification_lock_check_wrlock(segment_num)); + /* WARNING: resetting the obj will remove the WB flag. Make sure you either + * re-add it or remove it from lists where it was added based on the flag. */ + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); struct list_s *list = pseg->modified_old_objects; struct stm_undo_s *undo = (struct stm_undo_s *)list->items; @@ -1310,12 +1346,10 @@ free_bk(undo); if (only_obj != NULL) { + /* WB_EXECUTED should be set on small objs, but not on card objs */ assert(IMPLY(only_obj != NULL, (((struct object_s *)dst)->stm_flags - & (GCFLAG_NO_CONFLICT - | GCFLAG_WRITE_BARRIER - | GCFLAG_WB_EXECUTED)) - == (GCFLAG_NO_CONFLICT | GCFLAG_WRITE_BARRIER))); + & GCFLAG_NO_CONFLICT))); /* copy last element over this one */ end--; list->count -= 3; diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -1,6 +1,11 @@ import cffi, weakref from common import parent_dir, source_files +import os +# enable some malloc debug checks: +os.environ["MALLOC_CHECK_"] = "3" + + # ---------- ffi = cffi.FFI() @@ -668,6 +673,7 @@ force_generic_engine=True) + WORD = 8 HDR = lib.SIZEOF_MYOBJ assert HDR == 8 diff --git a/c8/test/test_noconfl.py b/c8/test/test_noconfl.py --- a/c8/test/test_noconfl.py +++ b/c8/test/test_noconfl.py @@ -153,3 +153,56 @@ self.switch(1, False) stm_validate() assert stm_get_char(o) == 'a' + + def test_obj_reset_on_validate_like_it_was_never_written_to(self): + get_card_value = lib._stm_get_card_value + + # if noconfl objs are reset during stm_validate(), their WB flag gets + # lost. make sure it is not in any of the lists where the WB flag is + # required / expected. + self.start_transaction() + o = stm_allocate_noconflict(16) + oh = stm_allocate_noconflict(1000+20*CARD_SIZE) + self.push_root(o) + self.push_root(oh) + self.commit_transaction() + + self.start_transaction() + oh = self.pop_root() + o = self.pop_root() + self.push_root(o) + self.push_root(oh) + + stm_set_char(o, 'a') + stm_set_char(oh, 'x', use_cards=True) + assert o in modified_old_objects() + assert oh in modified_old_objects() + assert o in objects_pointing_to_nursery() + assert oh not in objects_pointing_to_nursery() + assert get_card_value(oh, 0) == CARD_MARKED + assert oh in old_objects_with_cards_set() + + self.switch(1) + self.start_transaction() + stm_set_char(o, 'b') + stm_set_char(oh, 'y', use_cards=True) + self.commit_transaction() + + self.switch(0, False) + assert stm_get_char(o) == 'a' + assert stm_get_char(oh) == 'x' + assert o in modified_old_objects() + assert oh in modified_old_objects() + assert o in objects_pointing_to_nursery() + assert oh not in objects_pointing_to_nursery() + assert oh in old_objects_with_cards_set() + stm_validate() + assert stm_get_char(o) == 'b' + assert stm_get_char(oh) == 'y' + assert o not in modified_old_objects() + assert oh not in modified_old_objects() + assert o not in objects_pointing_to_nursery() + assert oh not in objects_pointing_to_nursery() + assert get_card_value(oh, 0) == CARD_CLEAR + assert oh not in old_objects_with_cards_set() + stm_minor_collect() From pypy.commits at gmail.com Wed Jun 29 12:22:11 2016 From: pypy.commits at gmail.com (Raemi) Date: Wed, 29 Jun 2016 09:22:11 -0700 (PDT) Subject: [pypy-commit] stmgc default: increase default NB_SEGMENTS from 4 to 8 Message-ID: <5773f5b3.d48e1c0a.9673c.ffffdada@mx.google.com> Author: Remi Meier Branch: Changeset: r1991:08d59f4f6b40 Date: 2016-06-29 13:28 +0200 http://bitbucket.org/pypy/stmgc/changeset/08d59f4f6b40/ Log: increase default NB_SEGMENTS from 4 to 8 diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -205,7 +205,7 @@ threads than the number of segments, it will block, waiting for the next segment to become free. */ -#define STM_NB_SEGMENTS 4 +#define STM_NB_SEGMENTS 8 /* Structure of objects -------------------- From pypy.commits at gmail.com Wed Jun 29 12:22:15 2016 From: pypy.commits at gmail.com (Raemi) Date: Wed, 29 Jun 2016 09:22:15 -0700 (PDT) Subject: [pypy-commit] stmgc default: add test that shows another bug with noconfl objs Message-ID: <5773f5b7.54df1c0a.8969e.5c72@mx.google.com> Author: Remi Meier Branch: Changeset: r1993:8d9da3988633 Date: 2016-06-29 18:21 +0200 http://bitbucket.org/pypy/stmgc/changeset/8d9da3988633/ Log: add test that shows another bug with noconfl objs Since stm_validate() may remove bk_cpys from the modified_objs list, it can change the commit log entry that gets added to the CL. But currently, we create the CL entry only once, possibly do a validation step, but then attach the old CL entry to the CL -> wrong (I found doubly freed bk_cpys). Instead, recalculate the CL entry after validation (if it did something) diff --git a/c8/test/test_noconfl.py b/c8/test/test_noconfl.py --- a/c8/test/test_noconfl.py +++ b/c8/test/test_noconfl.py @@ -206,3 +206,35 @@ assert get_card_value(oh, 0) == CARD_CLEAR assert oh not in old_objects_with_cards_set() stm_minor_collect() + + def test_reset_obj_during_commit_process(self): + self.start_transaction() + o = stm_allocate_noconflict(16) + self.push_root(o) + self.commit_transaction() + assert count_commit_log_entries() == 1 + + self.start_transaction() + o = self.pop_root() + self.push_root(o) + stm_set_char(o, 'a') + + self.switch(1) + self.start_transaction() + stm_set_char(o, 'b') + self.commit_transaction() + + assert count_commit_log_entries() == 2 + assert o in last_commit_log_entry_objs() + + self.switch(0, False) + assert stm_get_char(o) == 'a' + self.commit_transaction() # validates and succeeds! + + # the changes we did to our noconfl obj obviously must not appear in + # the commit log (since they get reverted by the validation step during + # commit). However, there was a bug that constructed the list of + # changes only once and does not account for validation to remove + # changes from that list. + assert count_commit_log_entries() == 3 + assert o not in last_commit_log_entry_objs() From pypy.commits at gmail.com Wed Jun 29 12:29:41 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 09:29:41 -0700 (PDT) Subject: [pypy-commit] pypy py3k: Use space.newbytes() and space.newunicode() Message-ID: <5773f775.081ac20a.870d9.ffffb29e@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85454:78b02e3ea03e Date: 2016-06-29 17:29 +0100 http://bitbucket.org/pypy/pypy/changeset/78b02e3ea03e/ Log: Use space.newbytes() and space.newunicode() diff too long, truncating to 2000 out of 2321 lines diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py --- a/pypy/interpreter/main.py +++ b/pypy/interpreter/main.py @@ -19,7 +19,7 @@ def compilecode(space, source, filename, cmd='exec'): w = space.wrap w_code = space.builtin.call( - 'compile', space.wrapbytes(source), space.wrap_fsdecoded(filename), + 'compile', space.newbytes(source), space.wrap_fsdecoded(filename), w(cmd), w(0), w(0)) pycode = space.interp_w(eval.Code, w_code) return pycode diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -420,14 +420,14 @@ w(self.co_nlocals), w(self.co_stacksize), w(self.co_flags), - space.wrapbytes(self.co_code), + space.newbytes(self.co_code), space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), w(self.co_firstlineno), - space.wrapbytes(self.co_lnotab), + space.newbytes(self.co_lnotab), space.newtuple([w(v) for v in self.co_freevars]), space.newtuple([w(v) for v in self.co_cellvars]), w(self.magic), @@ -448,7 +448,7 @@ space = self.space # co_name should be an identifier name = self.co_name.decode('utf-8') - fn = space.fsdecode_w(space.wrapbytes(self.co_filename)) + fn = space.fsdecode_w(space.newbytes(self.co_filename)) return space.wrap(u'' % ( name, unicode(self.getaddrstring(space)), fn, -1 if self.co_firstlineno == 0 else self.co_firstlineno)) diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -25,7 +25,7 @@ 'replace') w_text = space.wrap(text) if self.filename is not None: - w_filename = space.fsdecode(space.wrapbytes(self.filename)) + w_filename = space.fsdecode(space.newbytes(self.filename)) return space.newtuple([space.wrap(self.msg), space.newtuple([w_filename, space.wrap(self.lineno), diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -85,13 +85,13 @@ if rawmode or '\\' not in substr: if not unicode_literal: - return space.wrapbytes(substr) + return space.newbytes(substr) else: v = unicodehelper.decode_utf8(space, substr) return space.wrap(v) v = PyString_DecodeEscape(space, substr, 'strict', encoding) - return space.wrapbytes(v) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -7,7 +7,7 @@ def recode_to_utf8(space, bytes, encoding): if encoding == 'utf-8': return bytes - w_text = space.call_method(space.wrapbytes(bytes), "decode", + w_text = space.call_method(space.newbytes(bytes), "decode", space.wrap(encoding)) w_recoded = space.call_method(w_text, "encode", space.wrap("utf-8")) return space.bytes_w(w_recoded) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -345,13 +345,13 @@ def interp_attrproperty_bytes(name, cls, doc=None): "NOT_RPYTHON: initialization-time only" def fget(space, obj): - return space.wrapbytes(getattr(obj, name)) + return space.newbytes(getattr(obj, name)) return GetSetProperty(fget, cls=cls, doc=doc) def interp_attrproperty_fsdecode(name, cls, doc=None): "NOT_RPYTHON: initialization-time only" def fget(space, obj): - return space.fsdecode(space.wrapbytes(getattr(obj, name))) + return space.fsdecode(space.newbytes(getattr(obj, name))) return GetSetProperty(fget, cls=cls, doc=doc) def interp_attrproperty_w(name, cls, doc=None): diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -18,7 +18,7 @@ startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, space.newtuple([space.wrap(encoding), - space.wrapbytes(s), + space.newbytes(s), space.wrap(startingpos), space.wrap(endingpos), space.wrap(msg)])) @@ -111,7 +111,7 @@ return space.call_method(w_uni, 'encode', getfilesystemencoding(space), space.wrap('surrogateescape')) - return space.wrapbytes(bytes) + return space.newbytes(bytes) def encode(space, w_data, encoding=None, errors='strict'): from pypy.objspace.std.unicodeobject import encode_object diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -40,7 +40,7 @@ s = self.builder.build() self.builder = None if strtype is str: - return space.wrapbytes(s) + return space.newbytes(s) else: return space.wrap(s) diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -55,9 +55,9 @@ start, stop, step, size = space.decode_index4(w_index, self.buffer.getlength()) if step == 0: - return space.wrapbytes(self.buffer.getitem(start)) + return space.newbytes(self.buffer.getitem(start)) res = self.buffer.getslice(start, stop, step, size) - return space.wrapbytes(res) + return space.newbytes(res) def descr_setitem(self, space, w_index, w_newstring): start, stop, step, size = space.decode_index4(w_index, diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -84,7 +84,7 @@ if self.size == 1: with cdataobj as ptr: s = ptr[0] - return self.space.wrapbytes(s) + return self.space.newbytes(s) return W_CType.string(self, cdataobj, maxlen) def unpack_ptr(self, w_ctypeptr, ptr, length): @@ -126,7 +126,7 @@ return self.space.wrap(ord(cdata[0])) def convert_to_object(self, cdata): - return self.space.wrapbytes(cdata[0]) + return self.space.newbytes(cdata[0]) def _convert_to_char(self, w_ob): space = self.space @@ -146,7 +146,7 @@ def unpack_ptr(self, w_ctypeptr, ptr, length): s = rffi.charpsize2str(ptr, length) - return self.space.wrapbytes(s) + return self.space.newbytes(s) # XXX explicitly use an integer type instead of lltype.UniChar here, diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -122,7 +122,7 @@ s = rffi.charp2str(ptr) else: s = rffi.charp2strn(ptr, length) - return space.wrapbytes(s) + return space.newbytes(s) # # pointer to a wchar_t: builds and returns a unicode if self.is_unichar_ptr_or_array(): @@ -131,7 +131,7 @@ u = rffi.wcharp2unicode(cdata) else: u = rffi.wcharp2unicoden(cdata, length) - return space.wrap(u) + return space.newunicode(u) # return W_CType.string(self, cdataobj, maxlen) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -36,7 +36,7 @@ w_errorhandler = lookup_error(space, errors) if decode: w_cls = space.w_UnicodeDecodeError - w_input = space.wrapbytes(input) + w_input = space.newbytes(input) else: w_cls = space.w_UnicodeEncodeError w_input = space.wrap(input) @@ -313,7 +313,7 @@ res += chr(0xe0 | (ch >> 12)) res += chr(0x80 | ((ch >> 6) & 0x3f)) res += chr(0x80 | (ch & 0x3f)) - return space.newtuple([space.wrapbytes(res), w_end]) + return space.newtuple([space.newbytes(res), w_end]) elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError): start = space.int_w(space.getattr(w_exc, space.wrap('start'))) obj = space.bytes_w(space.getattr(w_exc, space.wrap('object'))) @@ -355,7 +355,7 @@ # Not a UTF-8b surrogate, fail with original exception raise OperationError(space.type(w_exc), w_exc) res += chr(ch - 0xdc00) - return space.newtuple([space.wrapbytes(res), w_end]) + return space.newtuple([space.newbytes(res), w_end]) elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError): consumed = 0 start = space.int_w(space.getattr(w_exc, space.wrap('start'))) @@ -426,7 +426,7 @@ @unwrap_spec(errors='str_or_None') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) - return space.newtuple([space.wrapbytes(s), space.wrap(len(s))]) + return space.newtuple([space.newbytes(s), space.wrap(len(s))]) @unwrap_spec(errors=str) def decode(space, w_obj, w_encoding=None, errors='strict'): @@ -484,7 +484,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) wrap_encoder.func_name = rname globals()[name] = wrap_encoder @@ -544,7 +544,7 @@ result = runicode.unicode_encode_mbcs( uni, len(uni), errors, state.encode_error_handler, force_replace=False) - return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(string='bufferstr', errors='str_or_None', w_final=WrappedDefault(False)) @@ -569,7 +569,7 @@ result = runicode.unicode_encode_utf_8( uni, len(uni), errors, state.encode_error_handler, allow_surrogates=False) - return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(string='bufferstr', errors='str_or_None', w_final = WrappedDefault(False)) @@ -735,7 +735,7 @@ result = runicode.unicode_encode_charmap( uni, len(uni), errors, state.encode_error_handler, mapping) - return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(chars=unicode) @@ -833,11 +833,11 @@ state = space.fromcache(CodecState) result = runicode.unicode_encode_unicode_internal( uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) else: # special case for this codec: bytes are returned as is string = space.readbuf_w(w_uni).as_str() - return space.newtuple([space.wrapbytes(string), space.wrap(len(string))]) + return space.newtuple([space.newbytes(string), space.wrap(len(string))]) # ____________________________________________________________ # support for the "string escape" translation @@ -848,11 +848,11 @@ data = space.bytes_w(w_data) from pypy.objspace.std.bytesobject import string_escape_encode result = string_escape_encode(data, False) - return space.newtuple([space.wrapbytes(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) @unwrap_spec(errors='str_or_None') def escape_decode(space, w_data, errors='strict'): data = space.getarg_w('s#', w_data) from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, errors, None) - return space.newtuple([space.wrapbytes(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -111,7 +111,7 @@ def digest(self, space): "Return the digest value as a string of binary data." digest = self._digest(space) - return space.wrapbytes(digest) + return space.newbytes(digest) def hexdigest(self, space): "Return the digest value as a string of hexadecimal digits." diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -352,7 +352,7 @@ self._writer_reset_buf() def _write(self, space, data): - w_data = space.wrapbytes(data) + w_data = space.newbytes(data) while True: try: w_written = space.call_method(self.w_raw, "write", w_data) @@ -424,7 +424,7 @@ else: raise oefmt(space.w_ValueError, "read length must be positive or -1") - return space.wrapbytes(res) + return space.newbytes(res) @unwrap_spec(size=int) def peek_w(self, space, size=0): @@ -441,7 +441,7 @@ have = self._readahead() if have > 0: data = ''.join(self.buffer[self.pos:self.pos+have]) - return space.wrapbytes(data) + return space.newbytes(data) # Fill the buffer from the raw stream, and copy it to the result self._reader_reset_buf() @@ -451,7 +451,7 @@ size = 0 self.pos = 0 data = ''.join(self.buffer[:size]) - return space.wrapbytes(data) + return space.newbytes(data) @unwrap_spec(size=int) def read1_w(self, space, size): @@ -461,7 +461,7 @@ if size < 0: raise oefmt(space.w_ValueError, "read length must be positive") if size == 0: - return space.wrapbytes("") + return space.newbytes("") with self.lock: # Return up to n bytes. If at least one byte is buffered, we only @@ -489,7 +489,7 @@ endpos = self.pos + size data = ''.join(self.buffer[self.pos:endpos]) self.pos = endpos - return space.wrapbytes(data) + return space.newbytes(data) def _read_all(self, space): "Read all the file, don't update the cache" @@ -522,7 +522,7 @@ current_size += size if self.abs_pos != -1: self.abs_pos += size - return space.wrapbytes(builder.build()) + return space.newbytes(builder.build()) def _raw_read(self, space, buffer, start, length): length = intmask(length) @@ -653,11 +653,11 @@ else: pos = -1 if pos >= 0: - w_res = space.wrapbytes(''.join(self.buffer[self.pos:pos+1])) + w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1])) self.pos = pos + 1 return w_res if have == limit: - w_res = space.wrapbytes(''.join(self.buffer[self.pos:self.pos+have])) + w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have])) self.pos += have return w_res @@ -699,7 +699,7 @@ written += have if limit >= 0: limit -= have - return space.wrapbytes(''.join(chunks)) + return space.newbytes(''.join(chunks)) # ____________________________________________________ # Write methods @@ -943,7 +943,7 @@ w_writer = None @unwrap_spec(buffer_size=int) - def descr_init(self, space, w_reader, w_writer, + def descr_init(self, space, w_reader, w_writer, buffer_size=DEFAULT_BUFFER_SIZE): try: self.w_reader = W_BufferedReader(space) diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -76,12 +76,12 @@ def read_w(self, space, w_size=None): self._check_closed(space) size = convert_size(space, w_size) - return space.wrapbytes(self.read(size)) + return space.newbytes(self.read(size)) def readline_w(self, space, w_limit=None): self._check_closed(space) limit = convert_size(space, w_limit) - return space.wrapbytes(self.readline(limit)) + return space.newbytes(self.readline(limit)) def read1_w(self, space, w_size): return self.read_w(space, w_size) @@ -128,7 +128,7 @@ def getvalue_w(self, space): self._check_closed(space) - return space.wrapbytes(self.getvalue()) + return space.newbytes(self.getvalue()) def tell_w(self, space): self._check_closed(space) @@ -175,7 +175,7 @@ def getstate_w(self, space): self._check_closed(space) return space.newtuple([ - space.wrapbytes(self.getvalue()), + space.newbytes(self.getvalue()), space.wrap(self.tell()), self.getdict(space)]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -397,7 +397,7 @@ raise wrap_oserror(space, e, exception_name='w_IOError') - return space.wrapbytes(s) + return space.newbytes(s) def readinto_w(self, space, w_buffer): self._check_closed(space) @@ -441,7 +441,7 @@ break builder.append(chunk) total += len(chunk) - return space.wrapbytes(builder.build()) + return space.newbytes(builder.build()) if sys.platform == "win32": def _truncate(self, size): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -246,7 +246,7 @@ if read[-1] == '\n': break - return space.wrapbytes(builder.build()) + return space.newbytes(builder.build()) def readlines_w(self, space, w_hint=None): hint = convert_size(space, w_hint) @@ -361,7 +361,7 @@ if not data: break builder.append(data) - return space.wrapbytes(builder.build()) + return space.newbytes(builder.build()) W_RawIOBase.typedef = TypeDef( '_io._RawIOBase', W_IOBase.typedef, diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -160,7 +160,7 @@ w_buffer, w_flag = space.unpackiterable(w_state, 2) flag = space.r_longlong_w(w_flag) else: - w_buffer = space.wrapbytes("") + w_buffer = space.newbytes("") flag = 0 flag <<= 1 if self.pendingcr: @@ -832,7 +832,7 @@ while True: try: space.call_method(self.w_buffer, "write", - space.wrapbytes(pending_bytes)) + space.newbytes(pending_bytes)) except OperationError as e: if trap_eintr(space, e): continue @@ -861,7 +861,7 @@ space.call_method(self.w_decoder, "reset") else: space.call_method(self.w_decoder, "setstate", - space.newtuple([space.wrapbytes(""), + space.newtuple([space.newbytes(""), space.wrap(cookie.dec_flags)])) def _encoder_setstate(self, space, cookie): @@ -1010,7 +1010,7 @@ i = 0 while i < len(input): w_decoded = space.call_method(self.w_decoder, "decode", - space.wrapbytes(input[i])) + space.newbytes(input[i])) check_decoded(space, w_decoded) chars_decoded += len(space.unicode_w(w_decoded)) diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py --- a/pypy/module/_locale/interp_locale.py +++ b/pypy/module/_locale/interp_locale.py @@ -41,7 +41,7 @@ def charp2uni(space, s): "Convert a char* pointer to unicode according to the current locale" - w_bytes = space.wrapbytes(rffi.charp2str(s)) + w_bytes = space.newbytes(rffi.charp2str(s)) # XXX use mbstowcs() return space.call_method(w_bytes, "decode", space.wrap("utf-8")) diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -20,7 +20,7 @@ self.update(string) def digest_w(self): - return self.space.wrapbytes(self.digest()) + return self.space.newbytes(self.digest()) def hexdigest_w(self): return self.space.wrap(self.hexdigest()) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -83,12 +83,12 @@ return space.w_None except curses_error as e: raise convert_error(space, e) - return space.wrapbytes(result) + return space.newbytes(result) @unwrap_spec(s='bufferstr') def tparm(space, s, args_w): args = [space.int_w(a) for a in args_w] try: - return space.wrapbytes(_curses_tparm(s, args)) + return space.newbytes(_curses_tparm(s, args)) except curses_error as e: raise convert_error(space, e) diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -113,7 +113,7 @@ pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) assert 0 <= pos <= len(object) self.pending = object[pos:] - return space.wrapbytes(output) + return space.newbytes(output) @unwrap_spec(errors="str_or_None") diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -40,7 +40,7 @@ raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.wrapbytes(output), + return space.newtuple([space.newbytes(output), space.wrap(len(input))]) @@ -66,7 +66,7 @@ space.w_UnicodeDecodeError, space.newtuple([ space.wrap(name), - space.wrapbytes(input), + space.newbytes(input), space.wrap(e.start), space.wrap(e.end), space.wrap(e.reason)])) diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -123,9 +123,9 @@ space, self.BUFFER_SIZE, maxlength) try: if newbuf: - return space.wrapbytes(rffi.charpsize2str(newbuf, res)) + return space.newbytes(rffi.charpsize2str(newbuf, res)) else: - return space.wrapbytes(rffi.charpsize2str(self.buffer, res)) + return space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) @@ -139,7 +139,7 @@ space, length - offset, PY_SSIZE_T_MAX) try: if newbuf: - raise BufferTooShort(space, space.wrapbytes( + raise BufferTooShort(space, space.newbytes( rffi.charpsize2str(newbuf, res))) rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res)) finally: @@ -167,9 +167,9 @@ space, self.BUFFER_SIZE, PY_SSIZE_T_MAX) try: if newbuf: - w_received = space.wrapbytes(rffi.charpsize2str(newbuf, res)) + w_received = space.newbytes(rffi.charpsize2str(newbuf, res)) else: - w_received = space.wrapbytes(rffi.charpsize2str(self.buffer, res)) + w_received = space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) diff --git a/pypy/module/_rawffi/alt/test/test_type_converter.py b/pypy/module/_rawffi/alt/test/test_type_converter.py --- a/pypy/module/_rawffi/alt/test/test_type_converter.py +++ b/pypy/module/_rawffi/alt/test/test_type_converter.py @@ -12,14 +12,14 @@ handle_signed = handle_all handle_unsigned = handle_all handle_pointer = handle_all - handle_char = handle_all + handle_char = handle_all handle_unichar = handle_all handle_longlong = handle_all handle_char_p = handle_all handle_unichar_p = handle_all handle_float = handle_all handle_singlefloat = handle_all - + def handle_struct(self, w_ffitype, w_structinstance): self.lastval = w_structinstance @@ -119,12 +119,12 @@ def test_strings(self): # first, try automatic conversion from applevel - self.check(app_types.char_p, self.space.wrapbytes('foo'), 'foo') - self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') - self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') + self.check(app_types.char_p, self.space.newbytes('foo'), 'foo') + self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') + self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') # then, try to pass explicit pointers self.check(app_types.char_p, self.space.wrap(42), 42) - self.check(app_types.unichar_p, self.space.wrap(42), 42) + self.check(app_types.unichar_p, self.space.wrap(42), 42) @@ -136,7 +136,7 @@ get_signed = get_all get_unsigned = get_all get_pointer = get_all - get_char = get_all + get_char = get_all get_unichar = get_all get_longlong = get_all get_char_p = get_all @@ -144,7 +144,7 @@ get_float = get_all get_singlefloat = get_all get_unsigned_which_fits_into_a_signed = get_all - + def convert(self, w_ffitype, val): self.val = val return self.do_and_wrap(w_ffitype) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -181,7 +181,7 @@ start, stop = self.decodeslice(space, w_slice) ll_buffer = self.ll_buffer result = [ll_buffer[i] for i in range(start, stop)] - return space.wrapbytes(''.join(result)) + return space.newbytes(''.join(result)) def setslice(self, space, w_slice, w_value): start, stop = self.decodeslice(space, w_slice) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -432,7 +432,7 @@ res = func(add_arg, argdesc, rffi.VOIDP) return space.wrap(rffi.cast(lltype.Unsigned, res)) elif c == 'c': - return space.wrapbytes(func(add_arg, argdesc, ll_type)) + return space.newbytes(func(add_arg, argdesc, ll_type)) elif c == 'f' or c == 'd' or c == 'g': return space.wrap(float(func(add_arg, argdesc, ll_type))) else: @@ -573,7 +573,7 @@ s = rffi.charp2str(charp_addr) else: s = rffi.charp2strn(charp_addr, maxlength) - return space.wrapbytes(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2unicode(space, address, maxlength=-1): @@ -591,7 +591,7 @@ if maxlength == -1: return charp2string(space, address) s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength) - return space.wrapbytes(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2rawunicode(space, address, maxlength=-1): diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -19,7 +19,7 @@ res = rsocket.gethostname() except SocketError as e: raise converted_error(space, e) - return space.fsdecode(space.wrapbytes(res)) + return space.fsdecode(space.newbytes(res)) @unwrap_spec(host=str) def gethostbyname(space, host): @@ -224,7 +224,7 @@ buf = rsocket.inet_aton(ip) except SocketError as e: raise converted_error(space, e) - return space.wrapbytes(buf) + return space.newbytes(buf) @unwrap_spec(packed="bufferstr") def inet_ntoa(space, packed): @@ -249,7 +249,7 @@ buf = rsocket.inet_pton(family, ip) except SocketError as e: raise converted_error(space, e) - return space.wrapbytes(buf) + return space.newbytes(buf) @unwrap_spec(family=int, packed="bufferstr") def inet_ntop(space, family, packed): diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -37,7 +37,7 @@ path = addr.get_path() if _c.linux and len(path) > 0 and path[0] == '\x00': # Linux abstract namespace - return space.wrapbytes(path) + return space.newbytes(path) else: return space.wrap_fsdecoded(path) elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): @@ -345,7 +345,7 @@ except SocketError as e: raise converted_error(space, e) buflen = space.int_w(w_buflen) - return space.wrapbytes(self.sock.getsockopt(level, optname, buflen)) + return space.newbytes(self.sock.getsockopt(level, optname, buflen)) def gettimeout_w(self, space): """gettimeout() -> timeout @@ -384,7 +384,7 @@ data = self.sock.recv(buffersize, flags) except SocketError as e: raise converted_error(space, e) - return space.wrapbytes(data) + return space.newbytes(data) @unwrap_spec(buffersize='nonnegint', flags=int) def recvfrom_w(self, space, buffersize, flags=0): @@ -398,7 +398,7 @@ w_addr = addr_as_object(addr, self.sock.fd, space) else: w_addr = space.w_None - return space.newtuple([space.wrapbytes(data), w_addr]) + return space.newtuple([space.newbytes(data), w_addr]) except SocketError as e: raise converted_error(space, e) diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -36,12 +36,12 @@ def slice_w(space, ctx, start, end, w_default): if 0 <= start <= end: if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrapbytes(ctx._buffer.getslice(start, end, 1, + return space.newbytes(ctx._buffer.getslice(start, end, 1, end-start)) if isinstance(ctx, rsre_core.StrMatchContext): - return space.wrapbytes(ctx._string[start:end]) + return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr[start:end]) + return space.newunicode(ctx._unicodestr[start:end]) else: # unreachable raise SystemError @@ -346,15 +346,15 @@ strbuilder, unicodebuilder, last_pos, ctx.end) if use_builder: if strbuilder is not None: - return space.wrapbytes(strbuilder.build()), n + return space.newbytes(strbuilder.build()), n else: assert unicodebuilder is not None - return space.wrap(unicodebuilder.build()), n + return space.newunicode(unicodebuilder.build()), n else: if space.isinstance_w(w_string, space.w_unicode): - w_emptystr = space.wrap(u'') + w_emptystr = space.newunicode(u'') else: - w_emptystr = space.wrapbytes('') + w_emptystr = space.newbytes('') w_item = space.call_method(w_emptystr, 'join', space.newlist(sublist_w)) return w_item, n @@ -584,11 +584,11 @@ def fget_string(self, space): ctx = self.ctx if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrapbytes(ctx._buffer.as_str()) + return space.newbytes(ctx._buffer.as_str()) elif isinstance(ctx, rsre_core.StrMatchContext): - return space.wrapbytes(ctx._string) + return space.newbytes(ctx._string) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr) + return space.newunicode(ctx._unicodestr) else: raise SystemError diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -252,14 +252,14 @@ rffi.cast(rffi.UCHARP, buf.raw), n) if ok == 0 or ok == 1: return space.newtuple([ - space.wrapbytes(buf.str(n)), + space.newbytes(buf.str(n)), space.wrap(ok == 1), ]) else: ok = libssl_RAND_bytes( rffi.cast(rffi.UCHARP, buf.raw), n) if ok == 1: - return space.wrapbytes(buf.str(n)) + return space.newbytes(buf.str(n)) raise ssl_error(space, "", errcode=libssl_ERR_get_error()) @@ -404,7 +404,7 @@ elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: if space.is_none(w_buffer): - return space.wrapbytes('') + return space.newbytes('') else: return space.wrap(0) raise ssl_error(space, @@ -430,7 +430,7 @@ elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): if space.is_none(w_buffer): - return space.wrapbytes('') + return space.newbytes('') else: return space.wrap(0) else: @@ -455,7 +455,7 @@ rwbuffer.setslice(0, result) return space.wrap(count) else: - return space.wrapbytes(result) + return space.newbytes(result) def _get_socket(self, space): w_socket = self.w_socket() @@ -677,7 +677,7 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrapbytes(rffi.charpsize2str(buf, intmask(length))) + return space.newbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx @@ -716,7 +716,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrapbytes(rffi.charpsize2str(buf_ptr[0], length)) + return space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -935,7 +935,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - w_value = space.wrapbytes(rffi.charpsize2str(buf_ptr[0], length)) + w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) w_value = space.call_method(w_value, "decode", space.wrap("utf-8")) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1273,7 +1273,7 @@ w_ssl_socket, space.w_None, w_ctx) else: - w_servername = space.wrapbytes(rffi.charp2str(servername)) + w_servername = space.newbytes(rffi.charp2str(servername)) try: w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) @@ -1326,7 +1326,7 @@ rgc.add_memory_pressure(10 * 1024 * 1024) self.check_hostname = False self.register_finalizer(space) - + # Defaults libssl_SSL_CTX_set_verify(self.ctx, SSL_VERIFY_NONE, None) options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS @@ -1849,7 +1849,7 @@ if not path: return space.w_None else: - return fsdecode(space, space.wrapbytes(rffi.charp2str(path))) + return fsdecode(space, space.newbytes(rffi.charp2str(path))) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,7 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, + if not CertGetEnhancedKeyUsage(pCertCtx, flags, lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: @@ -120,7 +120,7 @@ pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx) if not pCertCtx: break - w_cert = space.wrapbytes( + w_cert = space.newbytes( rffi.charpsize2str(pCertCtx.c_pbCertEncoded, intmask(pCertCtx.c_cbCertEncoded))) w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType) @@ -162,7 +162,7 @@ pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx) if not pCrlCtx: break - w_crl = space.wrapbytes( + w_crl = space.newbytes( rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded, intmask(pCrlCtx.c_cbCrlEncoded))) w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -378,7 +378,7 @@ return space.newlist(l) else: # REG_BINARY and all other types - return space.wrapbytes(rffi.charpsize2str(buf, buflen)) + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name=str, typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -48,7 +48,7 @@ ] cls.w_test_data = w_test_data = space.wrap(test_data) w_btest = space.newtuple([space.wrap("Raw data"), - space.wrapbytes("binary\x00data"), + space.newbytes("binary\x00data"), space.wrap(_winreg.REG_BINARY)]) w_test_data.append(w_btest) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -242,11 +242,11 @@ """ size = self.len if size == 0: - return space.wrapbytes('') + return space.newbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() - return self.space.wrapbytes(s) + return self.space.newbytes(s) def descr_fromstring(self, space, w_s): """ fromstring(string) diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py --- a/pypy/module/binascii/interp_base64.py +++ b/pypy/module/binascii/interp_base64.py @@ -25,7 +25,7 @@ -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, ] def _transform(n): if n == -1: @@ -71,7 +71,7 @@ if leftbits != 0: raise_Error(space, "Incorrect padding") - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -110,4 +110,4 @@ res.append(table_b2a_base64[(leftchar & 0xf) << 2]) res.append(PAD) res.append('\n') - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py --- a/pypy/module/binascii/interp_hexlify.py +++ b/pypy/module/binascii/interp_hexlify.py @@ -26,7 +26,7 @@ for c in data: res.append(_value2char(ord(c) >> 4)) res.append(_value2char(ord(c) & 0xf)) - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -55,4 +55,4 @@ a = _char2value(space, hexstr[i]) b = _char2value(space, hexstr[i+1]) res.append(chr((a << 4) | b)) - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py --- a/pypy/module/binascii/interp_hqx.py +++ b/pypy/module/binascii/interp_hqx.py @@ -12,37 +12,37 @@ FAIL = 0x7d table_a2b_hqx = [ - #^@ ^A ^B ^C ^D ^E ^F ^G + #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #\b \t \n ^K ^L \r ^N ^O + #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, - #^P ^Q ^R ^S ^T ^U ^V ^W + #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ + #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - # ! " # $ % & ' + # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - #( ) * + , - . / + #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, - #0 1 2 3 4 5 6 7 + #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, - #8 9 : ; < = > ? + #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, - #@ A B C D E F G + #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, - #H I J K L M N O + #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, - #P Q R S T U V W + #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, - #X Y Z [ \ ] ^ _ + #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, - #` a b c d e f g + #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, - #h i j k l m n o + #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, - #p q r s t u v w + #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, - #x y z { | } ~ ^? + #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, @@ -98,7 +98,7 @@ else: if pending_bits > 0: raise_Incomplete(space, 'String has incomplete number of bytes') - return space.newtuple([space.wrapbytes(res.build()), space.wrap(done)]) + return space.newtuple([space.newbytes(res.build()), space.wrap(done)]) # ____________________________________________________________ @@ -129,7 +129,7 @@ if leftbits > 0: leftchar <<= (6 - leftbits) res.append(hqx_encoding[leftchar & 0x3f]) - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -161,7 +161,7 @@ if lastpushed < 0: raise_Error(space, 'String starts with the RLE code \\x90') res.append_multiple_char(chr(lastpushed), count) - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -198,7 +198,7 @@ # string that rledecode_hqx() would expand back to 'data', there are # some programs somewhere that would start failing obscurely in rare # cases. - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py --- a/pypy/module/binascii/interp_qp.py +++ b/pypy/module/binascii/interp_qp.py @@ -57,7 +57,7 @@ if header and c == '_': c = ' ' odata.append(c) - return space.wrapbytes(odata.build()) + return space.newbytes(odata.build()) # ____________________________________________________________ @@ -160,4 +160,4 @@ odata.append(c) inp += 1 - return space.wrapbytes(odata.build()) + return space.newbytes(odata.build()) diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py --- a/pypy/module/binascii/interp_uu.py +++ b/pypy/module/binascii/interp_uu.py @@ -55,7 +55,7 @@ remaining = length - res.getlength() if remaining > 0: res.append_multiple_char('\x00', remaining) - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -87,4 +87,4 @@ res.append(chr(0x20 + (C & 0x3F))) res.append('\n') - return space.wrapbytes(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -295,7 +295,7 @@ datasize = len(data) if datasize == 0: - return self.space.wrapbytes("") + return self.space.newbytes("") if not self.running: raise oefmt(self.space.w_ValueError, @@ -320,7 +320,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrapbytes(res) + return self.space.newbytes(res) def flush(self): if not self.running: @@ -340,7 +340,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrapbytes(res) + return self.space.newbytes(res) W_BZ2Compressor.typedef = TypeDef("_bz2.BZ2Compressor", __doc__ = W_BZ2Compressor.__doc__, @@ -417,7 +417,7 @@ raise oefmt(self.space.w_EOFError, "end of stream was already found") if data == '': - return self.space.wrapbytes('') + return self.space.newbytes('') in_bufsize = len(data) @@ -446,7 +446,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrapbytes(res) + return self.space.newbytes(res) W_BZ2Decompressor.typedef = TypeDef("_bz2.BZ2Decompressor", diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py --- a/pypy/module/bz2/test/test_bz2_compdecomp.py +++ b/pypy/module/bz2/test/test_bz2_compdecomp.py @@ -44,13 +44,13 @@ spaceconfig = dict(usemodules=('bz2', 'time', 'struct')) def setup_class(cls): - cls.w_TEXT = cls.space.wrapbytes(TEXT) + cls.w_TEXT = cls.space.newbytes(TEXT) if cls.runappdirect: cls.w_decompress = decompress else: @gateway.unwrap_spec(data=bytes) def decompress_w(space, data): - return space.wrapbytes(decompress(cls, data)) + return space.newbytes(decompress(cls, data)) cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w)) cls.w_HUGE_OK = cls.space.wrap(HUGE_OK) @@ -120,9 +120,9 @@ spaceconfig = dict(usemodules=('bz2', 'time', 'struct')) def setup_class(cls): - cls.w_TEXT = cls.space.wrapbytes(TEXT) - cls.w_DATA = cls.space.wrapbytes(DATA) - cls.w_BUGGY_DATA = cls.space.wrapbytes(BUGGY_DATA) + cls.w_TEXT = cls.space.newbytes(TEXT) + cls.w_DATA = cls.space.newbytes(DATA) + cls.w_BUGGY_DATA = cls.space.newbytes(BUGGY_DATA) cls.space.appexec([], """(): import warnings""") # Work around a recursion limit @@ -205,14 +205,14 @@ spaceconfig = dict(usemodules=('bz2', 'time')) def setup_class(cls): - cls.w_TEXT = cls.space.wrapbytes(TEXT) - cls.w_DATA = cls.space.wrapbytes(DATA) + cls.w_TEXT = cls.space.newbytes(TEXT) + cls.w_DATA = cls.space.newbytes(DATA) if cls.runappdirect: cls.w_decompress = decompress else: @gateway.unwrap_spec(data=bytes) def decompress_w(space, data): - return space.wrapbytes(decompress(cls, data)) + return space.newbytes(decompress(cls, data)) cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w)) cls.w_HUGE_OK = cls.space.wrap(HUGE_OK) diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py --- a/pypy/module/bz2/test/test_bz2_file.py +++ b/pypy/module/bz2/test/test_bz2_file.py @@ -42,7 +42,7 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.DATA_CRLF = DATA_CRLF + mod.DATA_CRLF = DATA_CRLF mod.create_temp_file = create_temp_file mod.decompress = decompress mod.create_broken_temp_file = create_broken_temp_file @@ -58,11 +58,11 @@ } def setup_class(cls): - cls.w_TEXT = cls.space.wrapbytes(TEXT) + cls.w_TEXT = cls.space.newbytes(TEXT) cls.DATA = DATA - cls.w_DATA = cls.space.wrapbytes(DATA) + cls.w_DATA = cls.space.newbytes(DATA) cls.DATA_CRLF = DATA_CRLF - cls.w_DATA_CRLF = cls.space.wrapbytes(DATA_CRLF) + cls.w_DATA_CRLF = cls.space.newbytes(DATA_CRLF) cls.temppath = str(py.test.ensuretemp("bz2").join("foo")) cls.w_temppath = cls.space.wrap(cls.temppath) if cls.runappdirect: @@ -78,14 +78,14 @@ @gateway.unwrap_spec(data=bytes) def decompress_w(space, data): - return space.wrapbytes(decompress(cls, data)) + return space.newbytes(decompress(cls, data)) cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w)) def create_broken_temp_file_w(): create_broken_temp_file(cls) cls.w_create_broken_temp_file = cls.space.wrap( gateway.interp2app(create_broken_temp_file_w)) - cls.w_random_data = cls.space.wrapbytes(RANDOM_DATA) + cls.w_random_data = cls.space.newbytes(RANDOM_DATA) cls.space.appexec([], """(): import warnings""") # Work around a recursion limit @@ -407,7 +407,7 @@ # has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos") -# +# # if has_cmdline_bunzip2: # def decompress(self, data): # pop = popen2.Popen3("bunzip2", capturestderr=1) @@ -418,7 +418,7 @@ # if pop.wait() != 0: # ret = bz2.decompress(data) # return ret -# +# # else: # # popen2.Popen3 doesn't exist on Windows, and even if it did, bunzip2 # # isn't available to run. diff --git a/pypy/module/bz2/test/test_large.py b/pypy/module/bz2/test/test_large.py --- a/pypy/module/bz2/test/test_large.py +++ b/pypy/module/bz2/test/test_large.py @@ -8,7 +8,7 @@ if not cls.runappdirect: py.test.skip("skipping this very slow test; try 'pypy-c -A'") largetest_bz2 = py.path.local(__file__).dirpath().join("largetest.bz2") - cls.w_compressed_data = cls.space.wrapbytes(largetest_bz2.read('rb')) + cls.w_compressed_data = cls.space.newbytes(largetest_bz2.read('rb')) def test_decompress(self): from bz2 import decompress diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -46,7 +46,7 @@ """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - w_s = space.wrapbytes(rffi.charpsize2str(char_p, length)) + w_s = space.newbytes(rffi.charpsize2str(char_p, length)) else: w_s = space.newint(length) w_buffer = space.call_function(space.w_bytearray, w_s) @@ -80,7 +80,7 @@ if space.isinstance_w(w_obj, space.w_bytearray): oldlen = space.len_w(w_obj) if newlen > oldlen: - space.call_method(w_obj, 'extend', space.wrapbytes('\x00' * (newlen - oldlen))) + space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -13,7 +13,7 @@ "PyCompilerFlags", (("cf_flags", rffi.INT),)) PyCompilerFlagsPtr = lltype.Ptr(PyCompilerFlags) -PyCF_MASK = (consts.CO_FUTURE_DIVISION | +PyCF_MASK = (consts.CO_FUTURE_DIVISION | consts.CO_FUTURE_ABSOLUTE_IMPORT | consts.CO_FUTURE_WITH_STATEMENT | consts.CO_FUTURE_PRINT_FUNCTION | @@ -94,7 +94,7 @@ Py_eval_input = 258 def compile_string(space, source, filename, start, flags=0): - w_source = space.wrapbytes(source) + w_source = space.newbytes(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -227,4 +227,4 @@ cf.c_cf_flags = rffi.cast(rffi.INT, flags) return result - + diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -19,7 +19,7 @@ try: w_readline = space.getattr(w_obj, space.wrap('readline')) except OperationError: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "argument must be a file, or have a readline() method.") n = rffi.cast(lltype.Signed, n) @@ -37,7 +37,7 @@ On success, return a new file object that is opened on the file given by filename, with a file mode given by mode, where mode has the same semantics as the standard C routine fopen(). On failure, return NULL.""" - w_filename = space.wrapbytes(rffi.charp2str(filename)) + w_filename = space.newbytes(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'open', w_filename, w_mode) diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -8,7 +8,7 @@ space.wrap((2, 7)))): py.test.skip("unsupported before Python 2.7") - w_hello = space.wrapbytes("hello") + w_hello = space.newbytes("hello") w_view = api.PyMemoryView_FromObject(w_hello) w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -13,7 +13,7 @@ rffi.free_charp(filename) rffi.free_charp(mode) - space.call_method(w_file, "write", space.wrapbytes("text")) + space.call_method(w_file, "write", space.newbytes("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -373,10 +373,10 @@ if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.wrapbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(rffi.charpsize2str(s, size)) w_encoding = space.wrap(rffi.charp2str(encoding)) if errors: - w_errors = space.wrapbytes(rffi.charp2str(errors)) + w_errors = space.newbytes(rffi.charp2str(errors)) else: w_errors = None return space.call_method(w_str, 'decode', w_encoding, w_errors) @@ -481,7 +481,7 @@ locale encoding. Use 'strict' error handler on Windows.""" - w_bytes = space.wrapbytes(rffi.charpsize2str(s, size)) + w_bytes = space.newbytes(rffi.charpsize2str(s, size)) return space.fsdecode(w_bytes) @@ -496,7 +496,7 @@ Use PyUnicode_DecodeFSDefaultAndSize() if you know the string length. Use 'strict' error handler on Windows.""" - w_bytes = space.wrapbytes(rffi.charp2str(s)) + w_bytes = space.newbytes(rffi.charp2str(s)) return space.fsdecode(w_bytes) @@ -516,7 +516,7 @@ @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): """Create a Unicode object from an UTF-8 encoded null-terminated char buffer""" - w_str = space.wrapbytes(rffi.charp2str(s)) + w_str = space.newbytes(rffi.charp2str(s)) return space.call_method(w_str, 'decode', space.wrap("utf-8")) @cpython_api([CONST_STRING], PyObject) @@ -594,7 +594,7 @@ encoded string s. Return NULL if an exception was raised by the codec. """ - w_s = space.wrapbytes(rffi.charpsize2str(s, size)) + w_s = space.newbytes(rffi.charpsize2str(s, size)) if errors: w_errors = space.wrap(rffi.charp2str(errors)) else: diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py --- a/pypy/module/exceptions/interp_exceptions.py +++ b/pypy/module/exceptions/interp_exceptions.py @@ -844,7 +844,7 @@ def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): # typechecking if space.isinstance_w(w_object, space.w_bytearray): - w_bytes = space.wrapbytes(space.bufferstr_w(w_object)) + w_bytes = space.newbytes(space.bufferstr_w(w_object)) else: w_bytes = w_object space.str_w(w_encoding) diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -111,7 +111,7 @@ if rv < 0: raise _get_error(space, "fcntl") arg = rffi.charpsize2str(ll_arg, len(arg)) - return space.wrapbytes(arg) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -225,7 +225,7 @@ if mutate_flag != 0: rwbuffer.setslice(0, arg) return space.wrap(rv) - return space.wrapbytes(arg) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -246,7 +246,7 @@ if rv < 0: raise _get_error(space, "ioctl") arg = rffi.charpsize2str(ll_arg, len(arg)) - return space.wrapbytes(arg) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -27,7 +27,7 @@ c = x & 0xff x >>= 8 d = x & 0xff - return space.wrapbytes(chr(a) + chr(b) + chr(c) + chr(d)) + return space.newbytes(chr(a) + chr(b) + chr(c) + chr(d)) def get_tag(space): """get_tag() -> string diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -29,7 +29,7 @@ by dump(data, file).""" m = StringMarshaller(space, space.int_w(w_version)) m.dump_w_obj(w_data) - return space.wrapbytes(m.get_value()) + return space.newbytes(m.get_value()) def load(space, w_f): """Read one value from the file 'f' and return it.""" @@ -82,7 +82,7 @@ def write(self, data): space = self.space - space.call_function(self.func, space.wrapbytes(data)) + space.call_function(self.func, space.newbytes(data)) class FileReader(AbstractReaderWriter): diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py --- a/pypy/module/marshal/test/test_marshalimpl.py +++ b/pypy/module/marshal/test/test_marshalimpl.py @@ -78,7 +78,7 @@ interp_marshal.marshal(space, w_obj, m) assert ''.join(m.seen) == expected # - u = interp_marshal.StringUnmarshaller(space, space.wrapbytes(expected)) + u = interp_marshal.StringUnmarshaller(space, space.newbytes(expected)) w_long = u.load_w_obj() assert space.eq_w(w_long, w_obj) diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -39,7 +39,7 @@ def readline(self): self.check_valid() - return self.space.wrapbytes(self.mmap.readline()) + return self.space.newbytes(self.mmap.readline()) @unwrap_spec(w_num=WrappedDefault(None)) def read(self, w_num): @@ -48,7 +48,7 @@ num = -1 else: num = self.space.int_w(w_num) - return self.space.wrapbytes(self.mmap.read(num)) + return self.space.newbytes(self.mmap.read(num)) def find(self, w_tofind, w_start=None, w_end=None): self.check_valid() @@ -185,13 +185,13 @@ return space.wrap(ord(self.mmap.getitem(start))) elif step == 1: if stop - start < 0: - return space.wrapbytes("") - return space.wrapbytes(self.mmap.getslice(start, length)) + return space.newbytes("") + return space.newbytes(self.mmap.getslice(start, length)) else: b = StringBuilder(length) for i in range(start, stop, step): b.append(self.mmap.getitem(i)) - return space.wrapbytes(b.build()) + return space.newbytes(b.build()) def descr_setitem(self, w_index, w_value): space = self.space diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -589,7 +589,7 @@ # Yes, supports only 8bit encodings translationmap = space.unicode_w( space.call_method( - space.wrapbytes(self.all_chars), "decode", + space.newbytes(self.all_chars), "decode", space.wrap(name), space.wrap("replace"))) if len(translationmap) != 256: diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py --- a/pypy/module/struct/formatiterator.py +++ b/pypy/module/struct/formatiterator.py @@ -140,7 +140,7 @@ @specialize.argtype(1) def appendobj(self, value): if isinstance(value, str): - self.result_w.append(self.space.wrapbytes(value)) + self.result_w.append(self.space.newbytes(value)) else: self.result_w.append(self.space.wrap(value)) diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py --- a/pypy/module/struct/interp_struct.py +++ b/pypy/module/struct/interp_struct.py @@ -55,7 +55,7 @@ @unwrap_spec(format=str) def pack(space, format, args_w): - return space.wrapbytes(_pack(space, format, args_w)) + return space.newbytes(_pack(space, format, args_w)) # XXX inefficient diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py --- a/pypy/module/termios/interp_termios.py +++ b/pypy/module/termios/interp_termios.py @@ -46,7 +46,7 @@ iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tup l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]] # last one need to be chosen carefully - cc_w = [space.wrapbytes(i) for i in cc] + cc_w = [space.newbytes(i) for i in cc] if lflag & rtermios.ICANON: cc_w[rtermios.VMIN] = space.wrap(ord(cc[rtermios.VMIN][0])) cc_w[rtermios.VTIME] = space.wrap(ord(cc[rtermios.VTIME][0])) diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py --- a/pypy/module/zipimport/interp_zipimport.py +++ b/pypy/module/zipimport/interp_zipimport.py @@ -287,7 +287,7 @@ filename = self._find_relative_path(filename) try: data = self.zip_file.read(filename) - return space.wrapbytes(data) + return space.newbytes(data) except (KeyError, OSError, BadZipfile): raise oefmt(space.w_IOError, "Error reading file") except RZlibError as e: diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py --- a/pypy/module/zlib/interp_zlib.py +++ b/pypy/module/zlib/interp_zlib.py @@ -62,7 +62,7 @@ rzlib.deflateEnd(stream) except rzlib.RZlibError as e: raise zlib_error(space, e.msg) - return space.wrapbytes(result) + return space.newbytes(result) @unwrap_spec(string='bufferstr', wbits="c_int", bufsize=int) @@ -84,7 +84,7 @@ rzlib.inflateEnd(stream) except rzlib.RZlibError as e: raise zlib_error(space, e.msg) - return space.wrapbytes(result) + return space.newbytes(result) class ZLibObject(W_Root): @@ -158,7 +158,7 @@ self.unlock() except rzlib.RZlibError as e: raise zlib_error(space, e.msg) - return space.wrapbytes(result) + return space.newbytes(result) @unwrap_spec(mode="c_int") def flush(self, space, mode=rzlib.Z_FINISH): @@ -187,7 +187,7 @@ self.unlock() except rzlib.RZlibError as e: raise zlib_error(space, e.msg) - return space.wrapbytes(result) + return space.newbytes(result) @unwrap_spec(level=int, method=int, wbits=int, memLevel=int, strategy=int) @@ -294,7 +294,7 @@ string, finished, unused_len = result self.eof = finished self._save_unconsumed_input(data, finished, unused_len) - return space.wrapbytes(string) + return space.newbytes(string) def flush(self, space, w_length=None): """ @@ -320,7 +320,7 @@ else: string, finished, unused_len = result self._save_unconsumed_input(data, finished, unused_len) - return space.wrapbytes(string) + return space.newbytes(string) @unwrap_spec(wbits=int) diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py --- a/pypy/module/zlib/test/test_zlib.py +++ b/pypy/module/zlib/test/test_zlib.py @@ -23,9 +23,9 @@ """ cls.w_zlib = cls.space.getbuiltinmodule('zlib') expanded = b'some bytes which will be compressed' - cls.w_expanded = cls.space.wrapbytes(expanded) - cls.w_compressed = cls.space.wrapbytes(zlib.compress(expanded)) - cls.w_LICENSE = cls.space.wrapbytes( + cls.w_expanded = cls.space.newbytes(expanded) + cls.w_compressed = cls.space.newbytes(zlib.compress(expanded)) + cls.w_LICENSE = cls.space.newbytes( py.path.local(pypy.__file__).dirpath().dirpath() .join('LICENSE').read()) diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -202,7 +202,7 @@ "NOT_RPYTHON" raise NotImplementedError - def wrapbytes(self, x): + def newbytes(self, x): return w_some_obj() def wrap(self, x): @@ -224,9 +224,6 @@ "NOT_RPYTHON" raise NotImplementedError - def wrapbytes(self, x): - return w_some_obj() - def _see_interp2app(self, interp2app): "NOT_RPYTHON" activation = interp2app._code.activation diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -626,8 +626,8 @@ l = space.listview_bytes(w_list) if l is not None: if len(l) == 1: - return space.wrapbytes(l[0]) - return space.wrapbytes(self._val(space).join(l)) + return space.newbytes(l[0]) + return space.newbytes(self._val(space).join(l)) return self._StringMethods_descr_join(space, w_list) def _join_return_one(self, space, w_obj): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1016,7 +1016,7 @@ unerase = staticmethod(unerase) def wrap(self, unwrapped): - return self.space.wrapbytes(unwrapped) + return self.space.newbytes(unwrapped) def unwrap(self, wrapped): return self.space.bytes_w(wrapped) @@ -1056,7 +1056,7 @@ return self.space.newlist_bytes(self.listview_bytes(w_dict)) def wrapkey(space, key): - return space.wrapbytes(key) + return space.newbytes(key) ##@jit.look_inside_iff(lambda self, w_dict: ## w_dict_unrolling_heuristic(w_dict)) diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -324,7 +324,7 @@ def unknown_fmtchar(self): space = self.space c = self.fmt[self.fmtpos - 1] - w_s = space.wrap(c) if do_unicode else space.wrapbytes(c) + w_s = space.newunicode(c) if do_unicode else space.newbytes(c) raise oefmt(space.w_ValueError, "unsupported format character %R (%s) at index %d", w_s, hex(ord(c)), self.fmtpos - 1) diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -133,7 +133,7 @@ "can't convert negative int to unsigned") except OverflowError: raise oefmt(space.w_OverflowError, "int too big to convert") - return space.wrapbytes(byte_string) + return space.newbytes(byte_string) def descr_round(self, space, w_ndigits=None): """Rounding an Integral returns itself. diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1948,7 +1948,7 @@ _none_value = None def wrap(self, stringval): - return self.space.wrapbytes(stringval) + return self.space.newbytes(stringval) def unwrap(self, w_string): return self.space.bytes_w(w_string) diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py --- a/pypy/objspace/std/marshal_impl.py +++ b/pypy/objspace/std/marshal_impl.py @@ -250,7 +250,7 @@ @unmarshaller(TYPE_STRING) def unmarshal_bytes(space, u, tc): - return space.wrapbytes(u.get_str()) + return space.newbytes(u.get_str()) @unmarshaller(TYPE_STRINGREF) def unmarshal_stringref(space, u, tc): diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -70,7 +70,7 @@ def descr_tobytes(self, space): self._check_released(space) - return space.wrapbytes(self.as_str()) + return space.newbytes(self.as_str()) def descr_tolist(self, space): self._check_released(space) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -340,7 +340,6 @@ def newbytes(self, s): return W_BytesObject(s) - wrapbytes = newbytes def newunicode(self, uni): return W_UnicodeObject(uni) diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -581,7 +581,7 @@ return space.wrap(uid) def _newobj(self, space, w_iterable): - """Make a new frozenset by taking ownership of 'w_iterable'.""" + """Make a new frozenset by taking ownership of 'w_iterable'.""" return W_FrozensetObject(space, w_iterable) @staticmethod @@ -1232,7 +1232,7 @@ return self.space.bytes_w(w_item) def wrap(self, item): - return self.space.wrapbytes(item) + return self.space.newbytes(item) def iter(self, w_set): return BytesIteratorImplementation(self.space, self, w_set) @@ -1462,7 +1462,7 @@ def next_entry(self): for key in self.iterator: - return self.space.wrapbytes(key) + return self.space.newbytes(key) else: return None diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py --- a/pypy/objspace/std/test/test_bytesobject.py +++ b/pypy/objspace/std/test/test_bytesobject.py @@ -4,30 +4,30 @@ pass def test_bytes_w(self): - assert self.space.bytes_w(self.space.wrapbytes("foo")) == "foo" + assert self.space.bytes_w(self.space.newbytes("foo")) == "foo" def test_equality(self): - w = self.space.wrapbytes + w = self.space.newbytes assert self.space.eq_w(w('abc'), w('abc')) assert not self.space.eq_w(w('abc'), w('def')) def test_order_cmp(self): space = self.space - w = space.wrapbytes + w = space.newbytes assert self.space.is_true(space.lt(w('a'), w('b'))) assert self.space.is_true(space.lt(w('a'), w('ab'))) assert self.space.is_true(space.le(w('a'), w('a'))) assert self.space.is_true(space.gt(w('a'), w(''))) def test_truth(self): - w = self.space.wrapbytes + w = self.space.newbytes assert self.space.is_true(w('non-empty')) assert not self.space.is_true(w('')) def test_getitem(self): space = self.space w = space.wrap - w_str = space.wrapbytes('abc') + w_str = space.newbytes('abc') assert space.eq_w(space.getitem(w_str, w(0)), w(ord('a'))) assert space.eq_w(space.getitem(w_str, w(-1)), w(ord('c'))) self.space.raises_w(space.w_IndexError, @@ -38,7 +38,7 @@ def test_slice(self): space = self.space w = space.wrap - wb = space.wrapbytes + wb = space.newbytes w_str = wb('abc') w_slice = space.newslice(w(0), w(0), space.w_None) @@ -67,7 +67,7 @@ return w_None = space.w_None w = space.wrap - wb = space.wrapbytes + wb = space.newbytes w_str = wb('hello') w_slice = space.newslice(w_None, w_None, w(1)) @@ -83,7 +83,7 @@ From pypy.commits at gmail.com Wed Jun 29 13:28:48 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 10:28:48 -0700 (PDT) Subject: [pypy-commit] pypy default: Backport space.newbytes() and space.newunicode() from py3k and reduce the size of the diff between the branches Message-ID: <57740550.cf2d1c0a.f8d38.fffff209@mx.google.com> Author: Ronan Lamy Branch: Changeset: r85455:9a242ffd6706 Date: 2016-06-29 18:28 +0100 http://bitbucket.org/pypy/pypy/changeset/9a242ffd6706/ Log: Backport space.newbytes() and space.newunicode() from py3k and reduce the size of the diff between the branches diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -579,7 +579,7 @@ def descr_str(self, space): if type(self) is W_BytesObject: return self - return wrapstr(space, self._value) + return W_BytesObject(self._value) def descr_hash(self, space): x = compute_hash(self._value) @@ -725,8 +725,8 @@ l = space.listview_bytes(w_list) if l is not None: if len(l) == 1: - return space.wrap(l[0]) - return space.wrap(self._val(space).join(l)) + return space.newbytes(l[0]) + return space.newbytes(self._val(space).join(l)) return self._StringMethods_descr_join(space, w_list) _StringMethods_descr_split = descr_split @@ -858,10 +858,6 @@ W_BytesObject.EMPTY = W_BytesObject('') -def wrapstr(space, s): - return W_BytesObject(s) - - W_BytesObject.typedef = TypeDef( "str", basestring_typedef, None, "read", __new__ = interp2app(W_BytesObject.descr_new), diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -16,7 +16,7 @@ from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.bufferobject import W_Buffer from pypy.objspace.std.bytearrayobject import W_BytearrayObject -from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject, wrapstr +from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject from pypy.objspace.std.complexobject import W_ComplexObject from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject from pypy.objspace.std.floatobject import W_FloatObject @@ -31,7 +31,7 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.tupleobject import W_AbstractTupleObject, W_TupleObject from pypy.objspace.std.typeobject import W_TypeObject, TypeCache -from pypy.objspace.std.unicodeobject import W_UnicodeObject, wrapunicode +from pypy.objspace.std.unicodeobject import W_UnicodeObject class StdObjSpace(ObjSpace): @@ -128,9 +128,6 @@ assert typedef is not None return self.fromcache(TypeCache).getorbuild(typedef) - def wrapbytes(self, x): - return wrapstr(self, x) - @specialize.argtype(1) def wrap(self, x): "Wraps the Python value 'x' into one of the wrapper classes." @@ -151,9 +148,9 @@ else: return self.newint(x) if isinstance(x, str): - return wrapstr(self, x) + return self.newbytes(x) if isinstance(x, unicode): - return wrapunicode(self, x) + return self.newunicode(x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, W_Root): @@ -323,6 +320,12 @@ def newbuffer(self, w_obj): return W_Buffer(w_obj) + def newbytes(self, s): + return W_BytesObject(s) + + def newunicode(self, uni): + return W_UnicodeObject(uni) + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py --- a/pypy/objspace/std/test/test_bytesobject.py +++ b/pypy/objspace/std/test/test_bytesobject.py @@ -3,31 +3,31 @@ def teardown_method(self, method): pass - def test_str_w(self): - assert self.space.str_w(self.space.wrap("foo")) == "foo" + def test_bytes_w(self): + assert self.space.bytes_w(self.space.newbytes("foo")) == "foo" def test_equality(self): - w = self.space.wrap + w = self.space.newbytes assert self.space.eq_w(w('abc'), w('abc')) assert not self.space.eq_w(w('abc'), w('def')) def test_order_cmp(self): space = self.space - w = space.wrap + w = space.newbytes assert self.space.is_true(space.lt(w('a'), w('b'))) assert self.space.is_true(space.lt(w('a'), w('ab'))) assert self.space.is_true(space.le(w('a'), w('a'))) assert self.space.is_true(space.gt(w('a'), w(''))) def test_truth(self): - w = self.space.wrap + w = self.space.newbytes assert self.space.is_true(w('non-empty')) assert not self.space.is_true(w('')) def test_getitem(self): space = self.space w = space.wrap - w_str = w('abc') + w_str = space.newbytes('abc') assert self.space.eq_w(space.getitem(w_str, w(0)), w('a')) assert self.space.eq_w(space.getitem(w_str, w(-1)), w('c')) self.space.raises_w(space.w_IndexError, @@ -38,25 +38,26 @@ def test_slice(self): space = self.space w = space.wrap - w_str = w('abc') + wb = space.newbytes + w_str = wb('abc') w_slice = space.newslice(w(0), w(0), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('')) w_slice = space.newslice(w(0), w(1), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('a')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('a')) w_slice = space.newslice(w(0), w(10), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc')) w_slice = space.newslice(space.w_None, space.w_None, space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc')) w_slice = space.newslice(space.w_None, w(-1), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('ab')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('ab')) w_slice = space.newslice(w(-1), space.w_None, space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('c')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('c')) def test_extended_slice(self): space = self.space @@ -66,23 +67,24 @@ return w_None = space.w_None w = space.wrap - w_str = w('hello') + wb = space.newbytes + w_str = wb('hello') w_slice = space.newslice(w_None, w_None, w(1)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('hello')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hello')) w_slice = space.newslice(w_None, w_None, w(-1)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('olleh')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('olleh')) w_slice = space.newslice(w_None, w_None, w(2)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('hlo')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hlo')) w_slice = space.newslice(w(1), w_None, w(2)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('el')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('el')) def test_listview_bytes(self): - w_str = self.space.wrap('abcd') - assert self.space.listview_bytes(w_str) == list("abcd") + w_bytes = self.space.newbytes('abcd') + assert self.space.listview_bytes(w_bytes) == list("abcd") class AppTestBytesObject: @@ -110,28 +112,29 @@ assert str(exc_info.value) == expected def test_split(self): - assert "".split() == [] - assert "".split('x') == [''] - assert " ".split() == [] - assert "a".split() == ['a'] + assert b"".split() == [] + assert b"".split(b'x') == [b''] + assert b" ".split() == [] + assert b"a".split() == [b'a'] assert "a".split("a", 1) == ['', ''] - assert " ".split(" ", 1) == ['', ''] - assert "aa".split("a", 2) == ['', '', ''] - assert " a ".split() == ['a'] - assert "a b c".split() == ['a','b','c'] - assert 'this is the split function'.split() == ['this', 'is', 'the', 'split', 'function'] - assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd'] - assert 'a|b|c|d'.split('|', 2) == ['a', 'b', 'c|d'] - assert 'a b c d'.split(None, 1) == ['a', 'b c d'] - assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d'] - assert 'a b c d'.split(None, 3) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.split(None, 4) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.split(None, 0) == ['a b c d'] - assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d'] - assert 'a b c d '.split() == ['a', 'b', 'c', 'd'] - assert 'a//b//c//d'.split('//') == ['a', 'b', 'c', 'd'] - assert 'endcase test'.split('test') == ['endcase ', ''] - raises(ValueError, 'abc'.split, '') + assert b" ".split(b" ", 1) == [b'', b''] + assert b"aa".split(b"a", 2) == [b'', b'', b''] + assert b" a ".split() == [b'a'] + assert b"a b c".split() == [b'a',b'b',b'c'] + assert b'this is the split function'.split() == [ + b'this', b'is', b'the', b'split', b'function'] + assert b'a|b|c|d'.split(b'|') == [b'a', b'b', b'c', b'd'] + assert b'a|b|c|d'.split(b'|', 2) == [b'a', b'b', b'c|d'] + assert b'a b c d'.split(None, 1) == [b'a', b'b c d'] + assert b'a b c d'.split(None, 2) == [b'a', b'b', b'c d'] + assert b'a b c d'.split(None, 3) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.split(None, 4) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.split(None, 0) == [b'a b c d'] + assert b'a b c d'.split(None, 2) == [b'a', b'b', b'c d'] + assert b'a b c d '.split() == [b'a', b'b', b'c', b'd'] + assert b'a//b//c//d'.split(b'//') == [b'a', b'b', b'c', b'd'] + assert b'endcase test'.split(b'test') == [b'endcase ', b''] + raises(ValueError, b'abc'.split, b'') def test_rsplit(self): assert "".rsplit() == [] @@ -141,100 +144,100 @@ assert " ".rsplit(" ", 1) == ['', ''] assert "aa".rsplit("a", 2) == ['', '', ''] assert " a ".rsplit() == ['a'] - assert "a b c".rsplit() == ['a','b','c'] + assert b"a b c".rsplit() == [b'a',b'b',b'c'] assert 'this is the rsplit function'.rsplit() == ['this', 'is', 'the', 'rsplit', 'function'] - assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd'] - assert 'a|b|c|d'.rsplit('|', 2) == ['a|b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 1) == ['a b c', 'd'] - assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 3) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 4) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 0) == ['a b c d'] - assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd'] - assert 'a b c d '.rsplit() == ['a', 'b', 'c', 'd'] - assert 'a//b//c//d'.rsplit('//') == ['a', 'b', 'c', 'd'] - assert 'endcase test'.rsplit('test') == ['endcase ', ''] - raises(ValueError, 'abc'.rsplit, '') + assert b'a|b|c|d'.rsplit(b'|') == [b'a', b'b', b'c', b'd'] + assert b'a|b|c|d'.rsplit(b'|', 2) == [b'a|b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 1) == [b'a b c', b'd'] + assert b'a b c d'.rsplit(None, 2) == [b'a b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 3) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 4) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 0) == [b'a b c d'] + assert b'a b c d'.rsplit(None, 2) == [b'a b', b'c', b'd'] + assert b'a b c d '.rsplit() == [b'a', b'b', b'c', b'd'] + assert b'a//b//c//d'.rsplit(b'//') == [b'a', b'b', b'c', b'd'] + assert b'endcase test'.rsplit(b'test') == [b'endcase ', b''] + raises(ValueError, b'abc'.rsplit, b'') def test_split_splitchar(self): assert "/a/b/c".split('/') == ['','a','b','c'] def test_title(self): - assert "brown fox".title() == "Brown Fox" - assert "!brown fox".title() == "!Brown Fox" - assert "bROWN fOX".title() == "Brown Fox" - assert "Brown Fox".title() == "Brown Fox" - assert "bro!wn fox".title() == "Bro!Wn Fox" + assert b"brown fox".title() == b"Brown Fox" + assert b"!brown fox".title() == b"!Brown Fox" + assert b"bROWN fOX".title() == b"Brown Fox" + assert b"Brown Fox".title() == b"Brown Fox" + assert b"bro!wn fox".title() == b"Bro!Wn Fox" def test_istitle(self): - assert "".istitle() == False - assert "!".istitle() == False - assert "!!".istitle() == False - assert "brown fox".istitle() == False - assert "!brown fox".istitle() == False - assert "bROWN fOX".istitle() == False - assert "Brown Fox".istitle() == True - assert "bro!wn fox".istitle() == False - assert "Bro!wn fox".istitle() == False - assert "!brown Fox".istitle() == False - assert "!Brown Fox".istitle() == True - assert "Brow&&&&N Fox".istitle() == True - assert "!Brow&&&&n Fox".istitle() == False + assert b"".istitle() == False + assert b"!".istitle() == False + assert b"!!".istitle() == False + assert b"brown fox".istitle() == False + assert b"!brown fox".istitle() == False + assert b"bROWN fOX".istitle() == False + assert b"Brown Fox".istitle() == True + assert b"bro!wn fox".istitle() == False + assert b"Bro!wn fox".istitle() == False + assert b"!brown Fox".istitle() == False + assert b"!Brown Fox".istitle() == True + assert b"Brow&&&&N Fox".istitle() == True + assert b"!Brow&&&&n Fox".istitle() == False def test_capitalize(self): - assert "brown fox".capitalize() == "Brown fox" - assert ' hello '.capitalize() == ' hello ' - assert 'Hello '.capitalize() == 'Hello ' - assert 'hello '.capitalize() == 'Hello ' - assert 'aaaa'.capitalize() == 'Aaaa' - assert 'AaAa'.capitalize() == 'Aaaa' + assert b"brown fox".capitalize() == b"Brown fox" + assert b' hello '.capitalize() == b' hello ' + assert b'Hello '.capitalize() == b'Hello ' + assert b'hello '.capitalize() == b'Hello ' + assert b'aaaa'.capitalize() == b'Aaaa' + assert b'AaAa'.capitalize() == b'Aaaa' def test_rjust(self): - s = "abc" + s = b"abc" assert s.rjust(2) == s assert s.rjust(3) == s - assert s.rjust(4) == " " + s - assert s.rjust(5) == " " + s - assert 'abc'.rjust(10) == ' abc' - assert 'abc'.rjust(6) == ' abc' - assert 'abc'.rjust(3) == 'abc' - assert 'abc'.rjust(2) == 'abc' - assert 'abc'.rjust(5, '*') == '**abc' # Python 2.4 + assert s.rjust(4) == b" " + s + assert s.rjust(5) == b" " + s + assert b'abc'.rjust(10) == b' abc' + assert b'abc'.rjust(6) == b' abc' + assert b'abc'.rjust(3) == b'abc' + assert b'abc'.rjust(2) == b'abc' + assert b'abc'.rjust(5, b'*') == b'**abc' # Python 2.4 raises(TypeError, 'abc'.rjust, 5, 'xx') def test_ljust(self): - s = "abc" + s = b"abc" assert s.ljust(2) == s assert s.ljust(3) == s - assert s.ljust(4) == s + " " - assert s.ljust(5) == s + " " - assert 'abc'.ljust(10) == 'abc ' - assert 'abc'.ljust(6) == 'abc ' - assert 'abc'.ljust(3) == 'abc' - assert 'abc'.ljust(2) == 'abc' - assert 'abc'.ljust(5, '*') == 'abc**' # Python 2.4 + assert s.ljust(4) == s + b" " + assert s.ljust(5) == s + b" " + assert b'abc'.ljust(10) == b'abc ' + assert b'abc'.ljust(6) == b'abc ' + assert b'abc'.ljust(3) == b'abc' + assert b'abc'.ljust(2) == b'abc' + assert b'abc'.ljust(5, b'*') == b'abc**' # Python 2.4 raises(TypeError, 'abc'.ljust, 6, '') def test_replace(self): - assert 'one!two!three!'.replace('!', '@', 1) == 'one at two!three!' - assert 'one!two!three!'.replace('!', '') == 'onetwothree' - assert 'one!two!three!'.replace('!', '@', 2) == 'one at two@three!' - assert 'one!two!three!'.replace('!', '@', 3) == 'one at two@three@' - assert 'one!two!three!'.replace('!', '@', 4) == 'one at two@three@' - assert 'one!two!three!'.replace('!', '@', 0) == 'one!two!three!' - assert 'one!two!three!'.replace('!', '@') == 'one at two@three@' - assert 'one!two!three!'.replace('x', '@') == 'one!two!three!' - assert 'one!two!three!'.replace('x', '@', 2) == 'one!two!three!' - assert 'abc'.replace('', '-') == '-a-b-c-' - assert 'abc'.replace('', '-', 3) == '-a-b-c' - assert 'abc'.replace('', '-', 0) == 'abc' - assert ''.replace('', '') == '' - assert ''.replace('', 'a') == 'a' - assert 'abc'.replace('ab', '--', 0) == 'abc' - assert 'abc'.replace('xy', '--') == 'abc' - assert '123'.replace('123', '') == '' - assert '123123'.replace('123', '') == '' - assert '123x123'.replace('123', '') == 'x' + assert b'one!two!three!'.replace(b'!', b'@', 1) == b'one at two!three!' + assert b'one!two!three!'.replace(b'!', b'') == b'onetwothree' + assert b'one!two!three!'.replace(b'!', b'@', 2) == b'one at two@three!' + assert b'one!two!three!'.replace(b'!', b'@', 3) == b'one at two@three@' + assert b'one!two!three!'.replace(b'!', b'@', 4) == b'one at two@three@' + assert b'one!two!three!'.replace(b'!', b'@', 0) == b'one!two!three!' + assert b'one!two!three!'.replace(b'!', b'@') == b'one at two@three@' + assert b'one!two!three!'.replace(b'x', b'@') == b'one!two!three!' + assert b'one!two!three!'.replace(b'x', b'@', 2) == b'one!two!three!' + assert b'abc'.replace(b'', b'-') == b'-a-b-c-' + assert b'abc'.replace(b'', b'-', 3) == b'-a-b-c' + assert b'abc'.replace(b'', b'-', 0) == b'abc' + assert b''.replace(b'', b'') == b'' + assert b''.replace(b'', b'a') == b'a' + assert b'abc'.replace(b'ab', b'--', 0) == b'abc' + assert b'abc'.replace(b'xy', b'--') == b'abc' + assert b'123'.replace(b'123', b'') == b'' + assert b'123123'.replace(b'123', b'') == b'' + assert b'123x123'.replace(b'123', b'') == b'x' def test_replace_buffer(self): assert 'one'.replace(buffer('o'), buffer('n'), 1) == 'nne' @@ -245,23 +248,23 @@ assert s.strip() == "a b" assert s.rstrip() == " a b" assert s.lstrip() == "a b " - assert 'xyzzyhelloxyzzy'.strip('xyz') == 'hello' - assert 'xyzzyhelloxyzzy'.lstrip('xyz') == 'helloxyzzy' - assert 'xyzzyhelloxyzzy'.rstrip('xyz') == 'xyzzyhello' + assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello' + assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy' + assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello' def test_zfill(self): - assert '123'.zfill(2) == '123' - assert '123'.zfill(3) == '123' - assert '123'.zfill(4) == '0123' - assert '+123'.zfill(3) == '+123' - assert '+123'.zfill(4) == '+123' - assert '+123'.zfill(5) == '+0123' - assert '-123'.zfill(3) == '-123' - assert '-123'.zfill(4) == '-123' - assert '-123'.zfill(5) == '-0123' - assert ''.zfill(3) == '000' - assert '34'.zfill(1) == '34' - assert '34'.zfill(4) == '0034' + assert b'123'.zfill(2) == b'123' + assert b'123'.zfill(3) == b'123' + assert b'123'.zfill(4) == b'0123' + assert b'+123'.zfill(3) == b'+123' + assert b'+123'.zfill(4) == b'+123' + assert b'+123'.zfill(5) == b'+0123' + assert b'-123'.zfill(3) == b'-123' + assert b'-123'.zfill(4) == b'-123' + assert b'-123'.zfill(5) == b'-0123' + assert b''.zfill(3) == b'000' + assert b'34'.zfill(1) == b'34' + assert b'34'.zfill(4) == b'0034' def test_center(self): s="a b" @@ -275,251 +278,251 @@ assert s.center(7) == " a b " assert s.center(8) == " a b " assert s.center(9) == " a b " - assert 'abc'.center(10) == ' abc ' - assert 'abc'.center(6) == ' abc ' - assert 'abc'.center(3) == 'abc' - assert 'abc'.center(2) == 'abc' - assert 'abc'.center(5, '*') == '*abc*' # Python 2.4 - raises(TypeError, 'abc'.center, 4, 'cba') - assert ' abc'.center(7) == ' abc ' + assert b'abc'.center(10) == b' abc ' + assert b'abc'.center(6) == b' abc ' + assert b'abc'.center(3) == b'abc' + assert b'abc'.center(2) == b'abc' + assert b'abc'.center(5, b'*') == b'*abc*' # Python 2.4 + raises(TypeError, b'abc'.center, 4, b'cba') + assert b' abc'.center(7) == b' abc ' def test_count(self): - assert "".count("x") ==0 - assert "".count("") ==1 - assert "Python".count("") ==7 - assert "ab aaba".count("ab") ==2 - assert 'aaa'.count('a') == 3 - assert 'aaa'.count('b') == 0 - assert 'aaa'.count('a', -1) == 1 - assert 'aaa'.count('a', -10) == 3 - assert 'aaa'.count('a', 0, -1) == 2 - assert 'aaa'.count('a', 0, -10) == 0 - assert 'ababa'.count('aba') == 1 + assert b"".count(b"x") ==0 + assert b"".count(b"") ==1 + assert b"Python".count(b"") ==7 + assert b"ab aaba".count(b"ab") ==2 + assert b'aaa'.count(b'a') == 3 + assert b'aaa'.count(b'b') == 0 + assert b'aaa'.count(b'a', -1) == 1 + assert b'aaa'.count(b'a', -10) == 3 + assert b'aaa'.count(b'a', 0, -1) == 2 + assert b'aaa'.count(b'a', 0, -10) == 0 + assert b'ababa'.count(b'aba') == 1 def test_startswith(self): - assert 'ab'.startswith('ab') is True - assert 'ab'.startswith('a') is True - assert 'ab'.startswith('') is True - assert 'x'.startswith('a') is False - assert 'x'.startswith('x') is True - assert ''.startswith('') is True - assert ''.startswith('a') is False - assert 'x'.startswith('xx') is False - assert 'y'.startswith('xx') is False + assert b'ab'.startswith(b'ab') is True + assert b'ab'.startswith(b'a') is True + assert b'ab'.startswith(b'') is True + assert b'x'.startswith(b'a') is False + assert b'x'.startswith(b'x') is True + assert b''.startswith(b'') is True + assert b''.startswith(b'a') is False + assert b'x'.startswith(b'xx') is False + assert b'y'.startswith(b'xx') is False def test_startswith_more(self): - assert 'ab'.startswith('a', 0) is True - assert 'ab'.startswith('a', 1) is False - assert 'ab'.startswith('b', 1) is True - assert 'abc'.startswith('bc', 1, 2) is False - assert 'abc'.startswith('c', -1, 4) is True + assert b'ab'.startswith(b'a', 0) is True + assert b'ab'.startswith(b'a', 1) is False + assert b'ab'.startswith(b'b', 1) is True + assert b'abc'.startswith(b'bc', 1, 2) is False + assert b'abc'.startswith(b'c', -1, 4) is True def test_startswith_too_large(self): - assert 'ab'.startswith('b', 1) is True - assert 'ab'.startswith('', 2) is True - assert 'ab'.startswith('', 3) is False - assert 'ab'.endswith('b', 1) is True - assert 'ab'.endswith('', 2) is True - assert 'ab'.endswith('', 3) is False + assert b'ab'.startswith(b'b', 1) is True + assert b'ab'.startswith(b'', 2) is True + assert b'ab'.startswith(b'', 3) is False + assert b'ab'.endswith(b'b', 1) is True + assert b'ab'.endswith(b'', 2) is True + assert b'ab'.endswith(b'', 3) is False def test_startswith_tuples(self): - assert 'hello'.startswith(('he', 'ha')) - assert not 'hello'.startswith(('lo', 'llo')) - assert 'hello'.startswith(('hellox', 'hello')) - assert not 'hello'.startswith(()) - assert 'helloworld'.startswith(('hellowo', 'rld', 'lowo'), 3) - assert not 'helloworld'.startswith(('hellowo', 'ello', 'rld'), 3) - assert 'hello'.startswith(('lo', 'he'), 0, -1) - assert not 'hello'.startswith(('he', 'hel'), 0, 1) - assert 'hello'.startswith(('he', 'hel'), 0, 2) - raises(TypeError, 'hello'.startswith, (42,)) + assert b'hello'.startswith((b'he', b'ha')) + assert not b'hello'.startswith((b'lo', b'llo')) + assert b'hello'.startswith((b'hellox', b'hello')) + assert not b'hello'.startswith(()) + assert b'helloworld'.startswith((b'hellowo', b'rld', b'lowo'), 3) + assert not b'helloworld'.startswith((b'hellowo', b'ello', b'rld'), 3) + assert b'hello'.startswith((b'lo', b'he'), 0, -1) + assert not b'hello'.startswith((b'he', b'hel'), 0, 1) + assert b'hello'.startswith((b'he', b'hel'), 0, 2) + raises(TypeError, b'hello'.startswith, (42,)) def test_endswith(self): - assert 'ab'.endswith('ab') is True - assert 'ab'.endswith('b') is True - assert 'ab'.endswith('') is True - assert 'x'.endswith('a') is False - assert 'x'.endswith('x') is True - assert ''.endswith('') is True - assert ''.endswith('a') is False - assert 'x'.endswith('xx') is False - assert 'y'.endswith('xx') is False + assert b'ab'.endswith(b'ab') is True + assert b'ab'.endswith(b'b') is True + assert b'ab'.endswith(b'') is True + assert b'x'.endswith(b'a') is False + assert b'x'.endswith(b'x') is True + assert b''.endswith(b'') is True + assert b''.endswith(b'a') is False + assert b'x'.endswith(b'xx') is False + assert b'y'.endswith(b'xx') is False def test_endswith_more(self): - assert 'abc'.endswith('ab', 0, 2) is True - assert 'abc'.endswith('bc', 1) is True - assert 'abc'.endswith('bc', 2) is False - assert 'abc'.endswith('b', -3, -1) is True + assert b'abc'.endswith(b'ab', 0, 2) is True + assert b'abc'.endswith(b'bc', 1) is True + assert b'abc'.endswith(b'bc', 2) is False + assert b'abc'.endswith(b'b', -3, -1) is True def test_endswith_tuple(self): - assert not 'hello'.endswith(('he', 'ha')) - assert 'hello'.endswith(('lo', 'llo')) - assert 'hello'.endswith(('hellox', 'hello')) - assert not 'hello'.endswith(()) - assert 'helloworld'.endswith(('hellowo', 'rld', 'lowo'), 3) - assert not 'helloworld'.endswith(('hellowo', 'ello', 'rld'), 3, -1) - assert 'hello'.endswith(('hell', 'ell'), 0, -1) - assert not 'hello'.endswith(('he', 'hel'), 0, 1) - assert 'hello'.endswith(('he', 'hell'), 0, 4) - raises(TypeError, 'hello'.endswith, (42,)) + assert not b'hello'.endswith((b'he', b'ha')) + assert b'hello'.endswith((b'lo', b'llo')) + assert b'hello'.endswith((b'hellox', b'hello')) + assert not b'hello'.endswith(()) + assert b'helloworld'.endswith((b'hellowo', b'rld', b'lowo'), 3) + assert not b'helloworld'.endswith((b'hellowo', b'ello', b'rld'), 3, -1) + assert b'hello'.endswith((b'hell', b'ell'), 0, -1) + assert not b'hello'.endswith((b'he', b'hel'), 0, 1) + assert b'hello'.endswith((b'he', b'hell'), 0, 4) + raises(TypeError, b'hello'.endswith, (42,)) def test_expandtabs(self): import sys - assert 'abc\rab\tdef\ng\thi'.expandtabs() == 'abc\rab def\ng hi' - assert 'abc\rab\tdef\ng\thi'.expandtabs(8) == 'abc\rab def\ng hi' - assert 'abc\rab\tdef\ng\thi'.expandtabs(4) == 'abc\rab def\ng hi' - assert 'abc\r\nab\tdef\ng\thi'.expandtabs(4) == 'abc\r\nab def\ng hi' - assert 'abc\rab\tdef\ng\thi'.expandtabs() == 'abc\rab def\ng hi' - assert 'abc\rab\tdef\ng\thi'.expandtabs(8) == 'abc\rab def\ng hi' - assert 'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == 'abc\r\nab\r\ndef\ng\r\nhi' + assert b'abc\rab\tdef\ng\thi'.expandtabs() == b'abc\rab def\ng hi' + assert b'abc\rab\tdef\ng\thi'.expandtabs(8) == b'abc\rab def\ng hi' + assert b'abc\rab\tdef\ng\thi'.expandtabs(4) == b'abc\rab def\ng hi' + assert b'abc\r\nab\tdef\ng\thi'.expandtabs(4) == b'abc\r\nab def\ng hi' + assert b'abc\rab\tdef\ng\thi'.expandtabs() == b'abc\rab def\ng hi' + assert b'abc\rab\tdef\ng\thi'.expandtabs(8) == b'abc\rab def\ng hi' + assert b'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == b'abc\r\nab\r\ndef\ng\r\nhi' - s = 'xy\t' - assert s.expandtabs() == 'xy ' + s = b'xy\t' + assert s.expandtabs() == b'xy ' - s = '\txy\t' - assert s.expandtabs() == ' xy ' - assert s.expandtabs(1) == ' xy ' - assert s.expandtabs(2) == ' xy ' - assert s.expandtabs(3) == ' xy ' + s = b'\txy\t' + assert s.expandtabs() == b' xy ' + assert s.expandtabs(1) == b' xy ' + assert s.expandtabs(2) == b' xy ' + assert s.expandtabs(3) == b' xy ' - assert 'xy'.expandtabs() == 'xy' - assert ''.expandtabs() == '' + assert b'xy'.expandtabs() == b'xy' + assert b''.expandtabs() == b'' - raises(OverflowError, "t\tt\t".expandtabs, sys.maxint) + raises(OverflowError, b"t\tt\t".expandtabs, sys.maxint) def test_expandtabs_overflows_gracefully(self): import sys if sys.maxint > (1 << 32): skip("Wrong platform") - raises((MemoryError, OverflowError), 't\tt\t'.expandtabs, sys.maxint) + raises((MemoryError, OverflowError), b't\tt\t'.expandtabs, sys.maxint) def test_expandtabs_0(self): assert 'x\ty'.expandtabs(0) == 'xy' assert 'x\ty'.expandtabs(-42) == 'xy' def test_splitlines(self): - s = "" + s = b"" assert s.splitlines() == [] assert s.splitlines() == s.splitlines(1) - s = "a + 4" - assert s.splitlines() == ['a + 4'] + s = b"a + 4" + assert s.splitlines() == [b'a + 4'] # The following is true if no newline in string. assert s.splitlines() == s.splitlines(1) - s = "a + 4\nb + 2" - assert s.splitlines() == ['a + 4', 'b + 2'] - assert s.splitlines(1) == ['a + 4\n', 'b + 2'] - s="ab\nab\n \n x\n\n\n" - assert s.splitlines() ==['ab', 'ab', ' ', ' x', '', ''] + s = b"a + 4\nb + 2" + assert s.splitlines() == [b'a + 4', b'b + 2'] + assert s.splitlines(1) == [b'a + 4\n', b'b + 2'] + s = b"ab\nab\n \n x\n\n\n" + assert s.splitlines() ==[b'ab', b'ab', b' ', b' x', b'', b''] assert s.splitlines() ==s.splitlines(0) - assert s.splitlines(1) ==['ab\n', 'ab\n', ' \n', ' x\n', '\n', '\n'] - s="\none\n\two\nthree\n\n" - assert s.splitlines() ==['', 'one', '\two', 'three', ''] - assert s.splitlines(1) ==['\n', 'one\n', '\two\n', 'three\n', '\n'] + assert s.splitlines(1) ==[b'ab\n', b'ab\n', b' \n', b' x\n', b'\n', b'\n'] + s = b"\none\n\two\nthree\n\n" + assert s.splitlines() ==[b'', b'one', b'\two', b'three', b''] + assert s.splitlines(1) ==[b'\n', b'one\n', b'\two\n', b'three\n', b'\n'] # Split on \r and \r\n too - assert '12\r34\r\n56'.splitlines() == ['12', '34', '56'] - assert '12\r34\r\n56'.splitlines(1) == ['12\r', '34\r\n', '56'] + assert b'12\r34\r\n56'.splitlines() == [b'12', b'34', b'56'] + assert b'12\r34\r\n56'.splitlines(1) == [b'12\r', b'34\r\n', b'56'] def test_find(self): - assert 'abcdefghiabc'.find('abc') == 0 - assert 'abcdefghiabc'.find('abc', 1) == 9 - assert 'abcdefghiabc'.find('def', 4) == -1 - assert 'abcdef'.find('', 13) == -1 - assert 'abcdefg'.find('def', 5, None) == -1 - assert 'abcdef'.find('d', 6, 0) == -1 - assert 'abcdef'.find('d', 3, 3) == -1 - raises(TypeError, 'abcdef'.find, 'd', 1.0) + assert b'abcdefghiabc'.find(b'abc') == 0 + assert b'abcdefghiabc'.find(b'abc', 1) == 9 + assert b'abcdefghiabc'.find(b'def', 4) == -1 + assert b'abcdef'.find(b'', 13) == -1 + assert b'abcdefg'.find(b'def', 5, None) == -1 + assert b'abcdef'.find(b'd', 6, 0) == -1 + assert b'abcdef'.find(b'd', 3, 3) == -1 + raises(TypeError, b'abcdef'.find, b'd', 1.0) def test_index(self): from sys import maxint - assert 'abcdefghiabc'.index('') == 0 - assert 'abcdefghiabc'.index('def') == 3 - assert 'abcdefghiabc'.index('abc') == 0 - assert 'abcdefghiabc'.index('abc', 1) == 9 - assert 'abcdefghiabc'.index('def', -4*maxint, 4*maxint) == 3 - assert 'abcdefgh'.index('def', 2, None) == 3 - assert 'abcdefgh'.index('def', None, None) == 3 - raises(ValueError, 'abcdefghiabc'.index, 'hib') - raises(ValueError, 'abcdefghiab'.index, 'abc', 1) - raises(ValueError, 'abcdefghi'.index, 'ghi', 8) - raises(ValueError, 'abcdefghi'.index, 'ghi', -1) - raises(TypeError, 'abcdefghijklmn'.index, 'abc', 0, 0.0) - raises(TypeError, 'abcdefghijklmn'.index, 'abc', -10.0, 30) + assert b'abcdefghiabc'.index(b'') == 0 + assert b'abcdefghiabc'.index(b'def') == 3 + assert b'abcdefghiabc'.index(b'abc') == 0 + assert b'abcdefghiabc'.index(b'abc', 1) == 9 + assert b'abcdefghiabc'.index(b'def', -4*maxint, 4*maxint) == 3 + assert b'abcdefgh'.index(b'def', 2, None) == 3 + assert b'abcdefgh'.index(b'def', None, None) == 3 + raises(ValueError, b'abcdefghiabc'.index, b'hib') + raises(ValueError, b'abcdefghiab'.index, b'abc', 1) + raises(ValueError, b'abcdefghi'.index, b'ghi', 8) + raises(ValueError, b'abcdefghi'.index, b'ghi', -1) + raises(TypeError, b'abcdefghijklmn'.index, b'abc', 0, 0.0) + raises(TypeError, b'abcdefghijklmn'.index, b'abc', -10.0, 30) def test_rfind(self): - assert 'abc'.rfind('', 4) == -1 - assert 'abcdefghiabc'.rfind('abc') == 9 - assert 'abcdefghiabc'.rfind('') == 12 - assert 'abcdefghiabc'.rfind('abcd') == 0 - assert 'abcdefghiabc'.rfind('abcz') == -1 - assert 'abc'.rfind('', 0) == 3 - assert 'abc'.rfind('', 3) == 3 - assert 'abcdefgh'.rfind('def', 2, None) == 3 + assert b'abc'.rfind(b'', 4) == -1 + assert b'abcdefghiabc'.rfind(b'abc') == 9 + assert b'abcdefghiabc'.rfind(b'') == 12 + assert b'abcdefghiabc'.rfind(b'abcd') == 0 + assert b'abcdefghiabc'.rfind(b'abcz') == -1 + assert b'abc'.rfind(b'', 0) == 3 + assert b'abc'.rfind(b'', 3) == 3 + assert b'abcdefgh'.rfind(b'def', 2, None) == 3 def test_rindex(self): from sys import maxint - assert 'abcdefghiabc'.rindex('') == 12 - assert 'abcdefghiabc'.rindex('def') == 3 - assert 'abcdefghiabc'.rindex('abc') == 9 - assert 'abcdefghiabc'.rindex('abc', 0, -1) == 0 - assert 'abcdefghiabc'.rindex('abc', -4*maxint, 4*maxint) == 9 - raises(ValueError, 'abcdefghiabc'.rindex, 'hib') - raises(ValueError, 'defghiabc'.rindex, 'def', 1) - raises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1) - raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8) - raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1) - raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', 0, 0.0) - raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', -10.0, 30) + assert b'abcdefghiabc'.rindex(b'') == 12 + assert b'abcdefghiabc'.rindex(b'def') == 3 + assert b'abcdefghiabc'.rindex(b'abc') == 9 + assert b'abcdefghiabc'.rindex(b'abc', 0, -1) == 0 + assert b'abcdefghiabc'.rindex(b'abc', -4*maxint, 4*maxint) == 9 + raises(ValueError, b'abcdefghiabc'.rindex, b'hib') + raises(ValueError, b'defghiabc'.rindex, b'def', 1) + raises(ValueError, b'defghiabc'.rindex, b'abc', 0, -1) + raises(ValueError, b'abcdefghi'.rindex, b'ghi', 0, 8) + raises(ValueError, b'abcdefghi'.rindex, b'ghi', 0, -1) + raises(TypeError, b'abcdefghijklmn'.rindex, b'abc', 0, 0.0) + raises(TypeError, b'abcdefghijklmn'.rindex, b'abc', -10.0, 30) def test_partition(self): - assert ('this is the par', 'ti', 'tion method') == \ - 'this is the partition method'.partition('ti') + assert (b'this is the par', b'ti', b'tion method') == \ + b'this is the partition method'.partition(b'ti') # from raymond's original specification - S = 'http://www.python.org' - assert ('http', '://', 'www.python.org') == S.partition('://') - assert ('http://www.python.org', '', '') == S.partition('?') - assert ('', 'http://', 'www.python.org') == S.partition('http://') - assert ('http://www.python.', 'org', '') == S.partition('org') + S = b'http://www.python.org' + assert (b'http', b'://', b'www.python.org') == S.partition(b'://') + assert (b'http://www.python.org', b'', b'') == S.partition(b'?') + assert (b'', b'http://', b'www.python.org') == S.partition(b'http://') + assert (b'http://www.python.', b'org', b'') == S.partition(b'org') - raises(ValueError, S.partition, '') + raises(ValueError, S.partition, b'') raises(TypeError, S.partition, None) def test_rpartition(self): - assert ('this is the rparti', 'ti', 'on method') == \ - 'this is the rpartition method'.rpartition('ti') + assert (b'this is the rparti', b'ti', b'on method') == \ + b'this is the rpartition method'.rpartition(b'ti') # from raymond's original specification - S = 'http://www.python.org' - assert ('http', '://', 'www.python.org') == S.rpartition('://') - assert ('', '', 'http://www.python.org') == S.rpartition('?') - assert ('', 'http://', 'www.python.org') == S.rpartition('http://') - assert ('http://www.python.', 'org', '') == S.rpartition('org') + S = b'http://www.python.org' + assert (b'http', b'://', b'www.python.org') == S.rpartition(b'://') + assert (b'', b'', b'http://www.python.org') == S.rpartition(b'?') + assert (b'', b'http://', b'www.python.org') == S.rpartition(b'http://') + assert (b'http://www.python.', b'org', b'') == S.rpartition(b'org') - raises(ValueError, S.rpartition, '') + raises(ValueError, S.rpartition, b'') raises(TypeError, S.rpartition, None) def test_split_maxsplit(self): - assert "/a/b/c".split('/', 2) == ['','a','b/c'] - assert "a/b/c".split("/") == ['a', 'b', 'c'] - assert " a ".split(None, 0) == ['a '] - assert " a ".split(None, 1) == ['a'] - assert " a a ".split(" ", 0) == [' a a '] - assert " a a ".split(" ", 1) == ['', 'a a '] + assert b"/a/b/c".split(b'/', 2) == [b'',b'a',b'b/c'] + assert b"a/b/c".split(b"/") == [b'a', b'b', b'c'] + assert b" a ".split(None, 0) == [b'a '] + assert b" a ".split(None, 1) == [b'a'] + assert b" a a ".split(b" ", 0) == [b' a a '] + assert b" a a ".split(b" ", 1) == [b'', b'a a '] def test_join(self): - assert ", ".join(['a', 'b', 'c']) == "a, b, c" - assert "".join([]) == "" - assert "-".join(['a', 'b']) == 'a-b' - text = 'text' - assert "".join([text]) == text - assert " -- ".join([text]) is text - raises(TypeError, ''.join, 1) - raises(TypeError, ''.join, [1]) - raises(TypeError, ''.join, [[1]]) + assert b", ".join([b'a', b'b', b'c']) == b"a, b, c" + assert b"".join([]) == b"" + assert b"-".join([b'a', b'b']) == b'a-b' + text = b'text' + assert b"".join([text]) is text + assert b" -- ".join([text]) is text + raises(TypeError, b''.join, 1) + raises(TypeError, b''.join, [1]) + raises(TypeError, b''.join, [[1]]) def test_unicode_join_str_arg_ascii(self): raises(UnicodeDecodeError, u''.join, ['\xc3\xa1']) @@ -576,68 +579,68 @@ return unicode("fooled you!") return self.it.next() - f = ('a\n', 'b\n', 'c\n') + f = (b'a\n', b'b\n', b'c\n') got = " - ".join(OhPhooey(f)) assert got == unicode("a\n - b\n - fooled you! - c\n") def test_lower(self): - assert "aaa AAA".lower() == "aaa aaa" - assert "".lower() == "" + assert b"aaa AAA".lower() == b"aaa aaa" + assert b"".lower() == b"" def test_upper(self): - assert "aaa AAA".upper() == "AAA AAA" - assert "".upper() == "" + assert b"aaa AAA".upper() == b"AAA AAA" + assert b"".upper() == b"" def test_isalnum(self): - assert "".isalnum() == False - assert "!Bro12345w&&&&n Fox".isalnum() == False - assert "125 Brown Foxes".isalnum() == False - assert "125BrownFoxes".isalnum() == True + assert b"".isalnum() == False + assert b"!Bro12345w&&&&n Fox".isalnum() == False + assert b"125 Brown Foxes".isalnum() == False + assert b"125BrownFoxes".isalnum() == True def test_isalpha(self): - assert "".isalpha() == False - assert "!Bro12345w&&&&nFox".isalpha() == False - assert "Brown Foxes".isalpha() == False - assert "125".isalpha() == False + assert b"".isalpha() == False + assert b"!Bro12345w&&&&nFox".isalpha() == False + assert b"Brown Foxes".isalpha() == False + assert b"125".isalpha() == False def test_isdigit(self): - assert "".isdigit() == False - assert "!Bro12345w&&&&nFox".isdigit() == False - assert "Brown Foxes".isdigit() == False - assert "125".isdigit() == True + assert b"".isdigit() == False + assert b"!Bro12345w&&&&nFox".isdigit() == False + assert b"Brown Foxes".isdigit() == False + assert b"125".isdigit() == True def test_isspace(self): - assert "".isspace() == False - assert "!Bro12345w&&&&nFox".isspace() == False - assert " ".isspace() == True - assert "\t\t\b\b\n".isspace() == False - assert "\t\t".isspace() == True - assert "\t\t\r\r\n".isspace() == True + assert b"".isspace() == False + assert b"!Bro12345w&&&&nFox".isspace() == False + assert b" ".isspace() == True + assert b"\t\t\b\b\n".isspace() == False + assert b"\t\t".isspace() == True + assert b"\t\t\r\r\n".isspace() == True def test_islower(self): - assert "".islower() == False - assert " ".islower() == False - assert "\t\t\b\b\n".islower() == False - assert "b".islower() == True - assert "bbb".islower() == True - assert "!bbb".islower() == True - assert "BBB".islower() == False - assert "bbbBBB".islower() == False + assert b"".islower() == False + assert b" ".islower() == False + assert b"\t\t\b\b\n".islower() == False + assert b"b".islower() == True + assert b"bbb".islower() == True + assert b"!bbb".islower() == True + assert b"BBB".islower() == False + assert b"bbbBBB".islower() == False def test_isupper(self): - assert "".isupper() == False - assert " ".isupper() == False - assert "\t\t\b\b\n".isupper() == False - assert "B".isupper() == True - assert "BBB".isupper() == True - assert "!BBB".isupper() == True - assert "bbb".isupper() == False - assert "BBBbbb".isupper() == False + assert b"".isupper() == False + assert b" ".isupper() == False + assert b"\t\t\b\b\n".isupper() == False + assert b"B".isupper() == True + assert b"BBB".isupper() == True + assert b"!BBB".isupper() == True + assert b"bbb".isupper() == False + assert b"BBBbbb".isupper() == False def test_swapcase(self): - assert "aaa AAA 111".swapcase() == "AAA aaa 111" - assert "".swapcase() == "" + assert b"aaa AAA 111".swapcase() == b"AAA aaa 111" + assert b"".swapcase() == b"" def test_translate(self): def maketrans(origin, image): @@ -650,25 +653,25 @@ tbl = ''.join(L) return tbl - table = maketrans('abc', 'xyz') - assert 'xyzxyz' == 'xyzabcdef'.translate(table, 'def') + table = maketrans(b'abc', b'xyz') + assert b'xyzxyz' == b'xyzabcdef'.translate(table, b'def') exc = raises(TypeError, "'xyzabcdef'.translate(memoryview(table), 'def')") assert str(exc.value) == 'expected a character buffer object' - table = maketrans('a', 'A') - assert 'Abc' == 'abc'.translate(table) - assert 'xyz' == 'xyz'.translate(table) - assert 'yz' == 'xyz'.translate(table, 'x') + table = maketrans(b'a', b'A') + assert b'Abc' == b'abc'.translate(table) + assert b'xyz' == b'xyz'.translate(table) + assert b'yz' == b'xyz'.translate(table, b'x') - raises(ValueError, 'xyz'.translate, 'too short', 'strip') - raises(ValueError, 'xyz'.translate, 'too short') - raises(ValueError, 'xyz'.translate, 'too long'*33) + raises(ValueError, b'xyz'.translate, b'too short', b'strip') + raises(ValueError, b'xyz'.translate, b'too short') + raises(ValueError, b'xyz'.translate, b'too long'*33) - assert 'yz' == 'xyz'.translate(None, 'x') # 2.6 + assert b'yz' == b'xyz'.translate(None, b'x') # 2.6 def test_iter(self): l=[] - for i in iter("42"): + for i in iter(b"42"): l.append(i) assert l == ['4','2'] @@ -692,10 +695,10 @@ assert repr(chr(2)) =="'\\x02'" def test_contains(self): - assert '' in 'abc' - assert 'a' in 'abc' - assert 'ab' in 'abc' - assert not 'd' in 'abc' + assert b'' in b'abc' + assert b'a' in b'abc' + assert b'ab' in b'abc' + assert not b'd' in b'abc' raises(TypeError, 'a'.__contains__, 1) def test_decode(self): @@ -715,17 +718,17 @@ assert hash('hello world!') & 0x7fffffff == 0x2f0bb411 def test_buffer(self): - x = "he" - x += "llo" + x = b"he" + x += b"llo" b = buffer(x) assert len(b) == 5 assert b[-1] == "o" - assert b[:] == "hello" - assert b[1:0] == "" + assert b[:] == b"hello" + assert b[1:0] == b"" raises(TypeError, "b[3] = 'x'") def test_getnewargs(self): - assert "foo".__getnewargs__() == ("foo",) + assert b"foo".__getnewargs__() == (b"foo",) def test_subclass(self): class S(str): @@ -775,24 +778,24 @@ import sys if sys.maxint > 2**31-1: skip("Wrong platform") - s = "a" * (2**16) - raises(OverflowError, s.replace, "", s) + s = b"a" * (2**16) + raises(OverflowError, s.replace, b"", s) def test_getslice(self): assert "foobar".__getslice__(4, 4321) == "ar" - s = "abc" - assert s[:] == "abc" - assert s[1:] == "bc" - assert s[:2] == "ab" - assert s[1:2] == "b" - assert s[-2:] == "bc" - assert s[:-1] == "ab" - assert s[-2:2] == "b" - assert s[1:-1] == "b" - assert s[-2:-1] == "b" + s = b"abc" + assert s[:] == b"abc" + assert s[1:] == b"bc" + assert s[:2] == b"ab" + assert s[1:2] == b"b" + assert s[-2:] == b"bc" + assert s[:-1] == b"ab" + assert s[-2:2] == b"b" + assert s[1:-1] == b"b" + assert s[-2:-1] == b"b" def test_no_len_on_str_iter(self): - iterable = "hello" + iterable = b"hello" raises(TypeError, len, iter(iterable)) def test___radd__(self): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -127,7 +127,7 @@ space = self.space w = space.wrap - w_l = self.space.newlist([w("a"),w("b")]) + w_l = space.newlist([w("a"),w("b")]) w_l.getitems = None w_d = space.call_method(space.w_dict, "fromkeys", w_l) @@ -136,8 +136,9 @@ def test_listview_bytes_dict(self): w = self.space.wrap + wb = self.space.newbytes w_d = self.space.newdict() - w_d.initialize_content([(w("a"), w(1)), (w("b"), w(2))]) + w_d.initialize_content([(wb("a"), w(1)), (wb("b"), w(2))]) assert self.space.listview_bytes(w_d) == ["a", "b"] def test_listview_unicode_dict(self): @@ -154,9 +155,10 @@ def test_keys_on_string_unicode_int_dict(self, monkeypatch): w = self.space.wrap - + wb = self.space.newbytes + w_d = self.space.newdict() - w_d.initialize_content([(w(1), w("a")), (w(2), w("b"))]) + w_d.initialize_content([(w(1), wb("a")), (w(2), wb("b"))]) w_l = self.space.call_method(w_d, "keys") assert sorted(self.space.listview_int(w_l)) == [1,2] @@ -166,7 +168,7 @@ monkeypatch.setattr(self.space, 'newlist', not_allowed) # w_d = self.space.newdict() - w_d.initialize_content([(w("a"), w(1)), (w("b"), w(6))]) + w_d.initialize_content([(wb("a"), w(1)), (wb("b"), w(6))]) w_l = self.space.call_method(w_d, "keys") assert sorted(self.space.listview_bytes(w_l)) == ["a", "b"] @@ -683,6 +685,7 @@ setattr(a, s, 123) assert holder.seen is s + class AppTestDictViews: def test_dictview(self): d = {1: 2, 3: 4} @@ -958,7 +961,7 @@ def test_empty_to_string(self): d = {} assert "EmptyDictStrategy" in self.get_strategy(d) - d["a"] = 1 + d[b"a"] = 1 assert "BytesDictStrategy" in self.get_strategy(d) class O(object): diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py --- a/pypy/objspace/std/test/test_liststrategies.py +++ b/pypy/objspace/std/test/test_liststrategies.py @@ -13,20 +13,22 @@ def test_check_strategy(self): space = self.space w = space.wrap + wb = space.newbytes assert isinstance(W_ListObject(space, []).strategy, EmptyListStrategy) - assert isinstance(W_ListObject(space, [w(1),w('a')]).strategy, ObjectListStrategy) + assert isinstance(W_ListObject(space, [w(1),wb('a')]).strategy, ObjectListStrategy) assert isinstance(W_ListObject(space, [w(1),w(2),w(3)]).strategy, IntegerListStrategy) - assert isinstance(W_ListObject(space, [w('a'), w('b')]).strategy, + assert isinstance(W_ListObject(space, [wb('a'), wb('b')]).strategy, BytesListStrategy) assert isinstance(W_ListObject(space, [w(u'a'), w(u'b')]).strategy, UnicodeListStrategy) - assert isinstance(W_ListObject(space, [w(u'a'), w('b')]).strategy, + assert isinstance(W_ListObject(space, [w(u'a'), wb('b')]).strategy, ObjectListStrategy) # mixed unicode and bytes def test_empty_to_any(self): space = self.space w = space.wrap + wb = space.newbytes l = W_ListObject(space, []) assert isinstance(l.strategy, EmptyListStrategy) l.append(w((1,3))) @@ -39,7 +41,7 @@ l = W_ListObject(space, []) assert isinstance(l.strategy, EmptyListStrategy) - l.append(w('a')) + l.append(wb('a')) assert isinstance(l.strategy, BytesListStrategy) l = W_ListObject(space, []) @@ -63,9 +65,10 @@ def test_string_to_any(self): l = W_ListObject(self.space, - [self.space.wrap('a'),self.space.wrap('b'),self.space.wrap('c')]) + [self.space.newbytes('a'), self.space.newbytes('b'), + self.space.newbytes('c')]) assert isinstance(l.strategy, BytesListStrategy) - l.append(self.space.wrap('d')) + l.append(self.space.newbytes('d')) assert isinstance(l.strategy, BytesListStrategy) l.append(self.space.wrap(3)) assert isinstance(l.strategy, ObjectListStrategy) @@ -91,6 +94,7 @@ def test_setitem(self): space = self.space w = space.wrap + wb = space.newbytes # This should work if test_listobject.py passes l = W_ListObject(space, [w('a'),w('b'),w('c')]) assert space.eq_w(l.getitem(0), w('a')) @@ -106,7 +110,7 @@ assert isinstance(l.strategy, ObjectListStrategy) # BytesStrategy to ObjectStrategy - l = W_ListObject(space, [w('a'),w('b'),w('c')]) + l = W_ListObject(space, [wb('a'),wb('b'),wb('c')]) assert isinstance(l.strategy, BytesListStrategy) l.setitem(0, w(2)) assert isinstance(l.strategy, ObjectListStrategy) @@ -126,6 +130,7 @@ def test_insert(self): space = self.space w = space.wrap + wb = space.newbytes # no change l = W_ListObject(space, [w(1),w(2),w(3)]) assert isinstance(l.strategy, IntegerListStrategy) @@ -133,7 +138,7 @@ assert isinstance(l.strategy, IntegerListStrategy) # BytesStrategy - l = W_ListObject(space, [w('a'),w('b'),w('c')]) + l = W_ListObject(space, [wb('a'),wb('b'),wb('c')]) assert isinstance(l.strategy, BytesListStrategy) l.insert(3, w(2)) assert isinstance(l.strategy, ObjectListStrategy) @@ -159,7 +164,7 @@ # EmptyStrategy l = W_ListObject(space, []) assert isinstance(l.strategy, EmptyListStrategy) - l.insert(0, w('a')) + l.insert(0, wb('a')) assert isinstance(l.strategy, BytesListStrategy) l = W_ListObject(space, []) @@ -187,6 +192,7 @@ def test_setslice(self): space = self.space w = space.wrap + wb = space.newbytes l = W_ListObject(space, []) assert isinstance(l.strategy, EmptyListStrategy) @@ -212,7 +218,7 @@ assert isinstance(l.strategy, ObjectListStrategy) # BytesStrategy to ObjectStrategy - l = W_ListObject(space, [w('a'), w('b'), w('c')]) + l = W_ListObject(space, [wb('a'), wb('b'), wb('c')]) assert isinstance(l.strategy, BytesListStrategy) l.setslice(0, 1, 2, W_ListObject(space, [w(1), w(2), w(3)])) assert isinstance(l.strategy, ObjectListStrategy) @@ -324,6 +330,7 @@ def test_empty_extend_with_any(self): space = self.space w = space.wrap + wb = space.newbytes empty = W_ListObject(space, []) assert isinstance(empty.strategy, EmptyListStrategy) @@ -332,7 +339,7 @@ empty = W_ListObject(space, []) assert isinstance(empty.strategy, EmptyListStrategy) - empty.extend(W_ListObject(space, [w("a"), w("b"), w("c")])) + empty.extend(W_ListObject(space, [wb("a"), wb("b"), wb("c")])) assert isinstance(empty.strategy, BytesListStrategy) empty = W_ListObject(space, []) @@ -578,9 +585,11 @@ assert not self.space.eq_w(l1, l2) def test_weird_rangelist_bug(self): - l = make_range_list(self.space, 1, 1, 3) + space = self.space + l = make_range_list(space, 1, 1, 3) # should not raise - assert l.descr_getslice(self.space, self.space.wrap(15), self.space.wrap(2222)).strategy == self.space.fromcache(EmptyListStrategy) + w_slice = space.newslice(space.wrap(15), space.wrap(2222), space.wrap(1)) + assert l.descr_getitem(space, w_slice).strategy == space.fromcache(EmptyListStrategy) def test_add_to_rangelist(self): l1 = make_range_list(self.space, 1, 1, 3) @@ -589,17 +598,17 @@ assert self.space.eq_w(l3, W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(4), self.space.wrap(5)])) def test_unicode(self): - l1 = W_ListObject(self.space, [self.space.wrap("eins"), self.space.wrap("zwei")]) + l1 = W_ListObject(self.space, [self.space.newbytes("eins"), self.space.newbytes("zwei")]) assert isinstance(l1.strategy, BytesListStrategy) - l2 = W_ListObject(self.space, [self.space.wrap(u"eins"), self.space.wrap(u"zwei")]) + l2 = W_ListObject(self.space, [self.space.newunicode(u"eins"), self.space.newunicode(u"zwei")]) assert isinstance(l2.strategy, UnicodeListStrategy) - l3 = W_ListObject(self.space, [self.space.wrap("eins"), self.space.wrap(u"zwei")]) + l3 = W_ListObject(self.space, [self.space.newbytes("eins"), self.space.newunicode(u"zwei")]) assert isinstance(l3.strategy, ObjectListStrategy) def test_listview_bytes(self): space = self.space assert space.listview_bytes(space.wrap(1)) == None - w_l = self.space.newlist([self.space.wrap('a'), self.space.wrap('b')]) + w_l = self.space.newlist([self.space.newbytes('a'), self.space.newbytes('b')]) assert space.listview_bytes(w_l) == ["a", "b"] def test_listview_unicode(self): @@ -641,13 +650,13 @@ def test_string_uses_newlist_bytes(self): space = self.space - w_s = space.wrap("a b c") + w_s = space.newbytes("a b c") space.newlist = None try: w_l = space.call_method(w_s, "split") - w_l2 = space.call_method(w_s, "split", space.wrap(" ")) + w_l2 = space.call_method(w_s, "split", space.newbytes(" ")) w_l3 = space.call_method(w_s, "rsplit") - w_l4 = space.call_method(w_s, "rsplit", space.wrap(" ")) + w_l4 = space.call_method(w_s, "rsplit", space.newbytes(" ")) finally: del space.newlist assert space.listview_bytes(w_l) == ["a", "b", "c"] @@ -707,7 +716,7 @@ def test_listview_bytes_list(self): space = self.space - w_l = W_ListObject(space, [space.wrap("a"), space.wrap("b")]) + w_l = W_ListObject(space, [space.newbytes("a"), space.newbytes("b")]) assert self.space.listview_bytes(w_l) == ["a", "b"] def test_listview_unicode_list(self): @@ -1034,6 +1043,13 @@ assert [(type(x), x) for x in space.unwrap(w_l)] == [ (int, 5), (float, 1.2), (int, 1), (float, 1.0)] + def test_stringstrategy_wraps_bytes(self): + space = self.space + wb = space.newbytes + l = W_ListObject(space, [wb('a'), wb('b')]) + w_item = l.getitem(0) + assert isinstance(w_item, space.StringObjectCls) + class TestW_ListStrategiesDisabled: spaceconfig = {"objspace.std.withliststrategies": False} diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -84,6 +84,7 @@ from pypy.objspace.std.floatobject import W_FloatObject w = self.space.wrap + wb = self.space.newbytes intstr = self.space.fromcache(IntegerSetStrategy) tmp_func = intstr.get_storage_from_list # test if get_storage_from_list is no longer used @@ -95,7 +96,7 @@ assert w_set.strategy is intstr assert intstr.unerase(w_set.sstorage) == {1:None, 2:None, 3:None} - w_list = W_ListObject(self.space, [w("1"), w("2"), w("3")]) + w_list = W_ListObject(self.space, [wb("1"), wb("2"), wb("3")]) w_set = W_SetObject(self.space) _initialize_set(self.space, w_set, w_list) assert w_set.strategy is self.space.fromcache(BytesSetStrategy) @@ -126,9 +127,10 @@ def test_listview_bytes_int_on_set(self): w = self.space.wrap + wb = self.space.newbytes w_a = W_SetObject(self.space) - _initialize_set(self.space, w_a, w("abcdefg")) + _initialize_set(self.space, w_a, wb("abcdefg")) assert sorted(self.space.listview_bytes(w_a)) == list("abcdefg") assert self.space.listview_int(w_a) is None @@ -439,7 +441,7 @@ self.s = s def __repr__(self): return repr(self.s) - + s = set([1, 2, 3]) s.add(A(s)) therepr = repr(s) @@ -460,7 +462,7 @@ assert therepr.endswith("])") inner = set(therepr[11:-2].split(", ")) assert inner == set(["1", "2", "3", "frozenset(...)"]) - + def test_keyerror_has_key(self): s = set() try: @@ -477,7 +479,7 @@ return int(id(self) & 0x7fffffff) s = H() f = set([s]) - print f + print(f) assert s in f f.remove(s) f.add(s) @@ -553,7 +555,7 @@ assert v1 == v2 else: assert False, 'Expected KeyError' - + def test_singleton_empty_frozenset(self): class Frozenset(frozenset): pass From pypy.commits at gmail.com Wed Jun 29 13:44:37 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 10:44:37 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Fixes, progress Message-ID: <57740905.871a1c0a.1a4b3.6a1f@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85456:820fae9f795f Date: 2016-06-29 19:45 +0200 http://bitbucket.org/pypy/pypy/changeset/820fae9f795f/ Log: Fixes, progress diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -5,7 +5,7 @@ from rpython.rtyper.annlowlevel import cast_gcref_to_instance from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import gateway, typedef, pycode +from pypy.interpreter import gateway, typedef, pycode, pytraceback class DBState: @@ -159,8 +159,12 @@ def compile(source, mode): space = dbstate.space compiler = space.createcompiler() - code = compiler.compile(source, '', mode, 0, - hidden_applevel=True) + dbstate.extend_syntax_with_dollar_num = True + try: + code = compiler.compile(source, '', mode, 0, + hidden_applevel=True) + finally: + dbstate.extend_syntax_with_dollar_num = False return code @@ -213,7 +217,7 @@ return space = dbstate.space try: - code = compile(expression, 'exec') + code = compile(expression, 'single') w_revdb_output = space.wrap(W_RevDBOutput(space)) w_displayhook = get_revdb_displayhook(space) space.sys.setdictvalue(space, 'stdout', w_revdb_output) @@ -225,22 +229,28 @@ frame.getdictscope()) except OperationError as operationerr: + # can't use sys.excepthook: it will likely try to do 'import + # traceback', which might not be doable without using I/O + tb = operationerr.get_traceback() + if tb is not None: + revdb.send_output("Traceback (most recent call last):\n") + while tb is not None: + if not isinstance(tb, pytraceback.PyTraceback): + revdb.send_output(" ??? %s\n" % tb) + break + show_frame(tb.frame, tb.get_lineno(), indent=' ') + tb = tb.next + + # set the sys.last_xxx attributes w_type = operationerr.w_type w_value = operationerr.get_w_value(space) - w_traceback = space.wrap(operationerr.get_traceback()) - - # set the sys.last_xxx attributes + w_tb = space.wrap(operationerr.get_traceback()) space.setitem(space.sys.w_dict, space.wrap('last_type'), w_type) space.setitem(space.sys.w_dict, space.wrap('last_value'), w_value) - space.setitem(space.sys.w_dict, space.wrap('last_traceback'), - w_traceback) + space.setitem(space.sys.w_dict, space.wrap('last_traceback'), w_tb) - # call sys.excepthook if present - w_hook = space.sys.getdictvalue(space, 'excepthook') - if w_hook is None: - raise - space.call_function(w_hook, w_type, w_value, w_traceback) - return + # re-raise, catch me in the outside "except OperationError" + raise except OperationError as e: revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) @@ -248,9 +258,10 @@ lambda_print = lambda: command_print -def show_frame(frame, indent=''): +def show_frame(frame, lineno=0, indent=''): code = frame.getcode() - lineno = frame.get_last_lineno() + if lineno == 0: + lineno = frame.get_last_lineno() revdb.send_output('%sFile "%s", line %d in %s\n%s ' % ( indent, code.co_filename, lineno, code.co_name, indent)) revdb.send_linecache(code.co_filename, lineno) @@ -262,7 +273,7 @@ if cmd.c_arg1 == 0: show_frame(frame) else: - revdb.send_output("Traceback (most recent call last):\n") + revdb.send_output("Current call stack (most recent call last):\n") frames = [] while frame is not None: frames.append(frame) diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -505,7 +505,8 @@ static uint64_t *future_ids, *future_next_id; static void *finalizer_tree, *destructor_tree; -static void attach_gdb(void) +RPY_EXTERN +void attach_gdb(void) { char cmdline[80]; sprintf(cmdline, "term -c \"gdb --pid=%d\"", getpid()); From pypy.commits at gmail.com Wed Jun 29 15:57:05 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 12:57:05 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <57742811.4f8d1c0a.45157.24ba@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85461:e0d9ed6c5aff Date: 2016-06-29 20:56 +0100 http://bitbucket.org/pypy/pypy/changeset/e0d9ed6c5aff/ Log: hg merge default From pypy.commits at gmail.com Wed Jun 29 16:00:55 2016 From: pypy.commits at gmail.com (rlamy) Date: Wed, 29 Jun 2016 13:00:55 -0700 (PDT) Subject: [pypy-commit] pypy py3k: fix translation Message-ID: <577428f7.81eac20a.de1f1.fffff1b1@mx.google.com> Author: Ronan Lamy Branch: py3k Changeset: r85462:306f1a7ca98e Date: 2016-06-29 21:00 +0100 http://bitbucket.org/pypy/pypy/changeset/306f1a7ca98e/ Log: fix translation diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -36,7 +36,7 @@ in frm is mapped to the byte at the same position in to. The bytes objects frm and to must be of the same length. """ - from pypy.objspace.std.bytesobject import makebytesdata_w, wrapstr + from pypy.objspace.std.bytesobject import makebytesdata_w base_table = [chr(i) for i in range(256)] list_from = makebytesdata_w(space, w_from) @@ -51,7 +51,7 @@ char_to = list_to[i] base_table[pos_from] = char_to - return wrapstr(space, ''.join(base_table)) + return space.newbytes(''.join(base_table)) def _multi_chr(self, c): return c From pypy.commits at gmail.com Wed Jun 29 17:48:16 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 14:48:16 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: tweak 'next' and 'bnext' to work with opaque stack frame IDs instead of Message-ID: <57744220.c72d1c0a.1515f.231e@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85463:17761780c45d Date: 2016-06-29 22:30 +0200 http://bitbucket.org/pypy/pypy/changeset/17761780c45d/ Log: tweak 'next' and 'bnext' to work with opaque stack frame IDs instead of the stack depth diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -14,13 +14,13 @@ CMD_BACKTRACE = 2 CMD_LOCALS = 3 CMD_BREAKPOINTS = 4 -CMD_MOREINFO = 5 +CMD_STACKID = 5 CMD_ATTACHID = 6 CMD_CHECKWATCH = 7 CMD_WATCHVALUES = 8 ANSWER_LINECACHE= 19 ANSWER_TEXT = 20 -ANSWER_MOREINFO = 21 +ANSWER_STACKID = 21 ANSWER_NEXTNID = 22 ANSWER_WATCH = 23 diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -196,51 +196,61 @@ command_bs = command_bstep @contextmanager - def _stack_depth_break(self, range_stop): - # add temporarily a breakpoint for "stack_depth < range_stop" + def _stack_id_break(self, stack_id): + # add temporarily a breakpoint that hits when we enter/leave + # the frame identified by 'stack_id' b = self.pgroup.edit_breakpoints() - b.stack_depth = range_stop + b.stack_id = stack_id try: yield finally: - b.stack_depth = 0 + b.stack_id = 0 def command_next(self, argument): """Run forward for one step, skipping calls""" - depth1 = self.pgroup.get_stack_depth() - if self.move_forward(1): - depth2 = self.pgroup.get_stack_depth() - if depth2 > depth1: - # If, after running one step, the stack depth is greater - # than before, then continue until it is back to what it was. - # Can't do it more directly because the "breakpoint" of - # stack_depth is only checked for on function enters and - # returns (which simplifies and speeds up things for the - # RPython code). - with self._stack_depth_break(depth1 + 1): - self.command_continue('') + depth1 = self.pgroup.get_stack_id() + with self._stack_id_break(depth1): + if not self.move_forward(1): + # we either hit a regular breakpoint, or we hit the + # temporary breakpoint + return + if depth1 == 0: # we started outside any frame + return + if self.pgroup.get_stack_id() == depth1: + return # we are still in the same frame + # + # If, after running one step, the stack id is different than + # before but we didn't leave that frame, then we must have + # entered a new one. Continue until we leave that new frame. + # Can't do it more directly because the "breakpoint" of + # stack_id is only checked for on function enters and returns + # (which simplifies and speeds up things for the RPython + # code). + self.command_finish('') command_n = command_next def command_bnext(self, argument): """Run backward for one step, skipping calls""" - depth1 = self.pgroup.get_stack_depth() - if self.move_backward(1): - depth2 = self.pgroup.get_stack_depth() - if depth2 > depth1: - # If, after running one bstep, the stack depth is greater - # than before, then bcontinue until it is back to what it was. - with self._stack_depth_break(depth1 + 1): - self.command_bcontinue('') + # similar to command_next() + depth1 = self.pgroup.get_stack_id() + with self._stack_id_break(depth1): + if not self.move_backward(1): + return + if depth1 == 0: + return + if self.pgroup.get_stack_id() == depth1: + return # we are still in the same frame + self.command_bfinish('') command_bn = command_bnext def command_finish(self, argument): """Run forward until the current function finishes""" - with self._stack_depth_break(self.pgroup.get_stack_depth()): + with self._stack_id_break(self.pgroup.get_stack_id()): self.command_continue('') def command_bfinish(self, argument): """Run backward until the current function is called""" - with self._stack_depth_break(self.pgroup.get_stack_depth()): + with self._stack_id_break(self.pgroup.get_stack_id()): self.command_bcontinue('') def command_continue(self, argument): diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -14,9 +14,9 @@ CMD_PRINT = 1 # Message(CMD_PRINT, extra=expression) CMD_BACKTRACE = 2 # Message(CMD_BACKTRACE) CMD_LOCALS = 3 # Message(CMD_LOCALS) -CMD_BREAKPOINTS = 4 # Message(CMD_BREAKPOINTS, stack_depth, +CMD_BREAKPOINTS = 4 # Message(CMD_BREAKPOINTS, stack_id, # extra="\0-separated names") -CMD_MOREINFO = 5 # Message(CMD_MOREINFO) +CMD_STACKID = 5 # Message(CMD_STACKID, parent-flag) CMD_ATTACHID = 6 # Message(CMD_ATTACHID, small-num, unique-id) CMD_CHECKWATCH = 7 # Message(CMD_CHECKWATCH, extra=expression) CMD_WATCHVALUES = 8 # Message(CMD_WATCHVALUES, extra=texts) @@ -55,9 +55,13 @@ # Message(ANSWER_TEXT, extra=text) ANSWER_TEXT = 20 -# sent after CMD_MOREINFO: -# Message(ANSWER_MOREINFO, stack_depth) -ANSWER_MOREINFO = 21 +# CMD_STACKID returns the id of the current or parent frame (depending +# on the 'parent-flag' passed in), or 0 if not found. The id can be just +# the stack depth, or it can be the unique id of the frame object. When +# used in CMD_BREAKPOINTS, it means "break if we are entering/leaving that +# frame". +# Message(ANSWER_STACKID, stack-id) +ANSWER_STACKID = 21 # sent from CMD_PRINT to record the existence of a recallable object # Message(ANSWER_NEXTNID, unique-id) diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -19,16 +19,16 @@ self.num2name = {} # {small number: break/watchpoint} self.watchvalues = {} # {small number: resulting text} self.watchuids = {} # {small number: [uid...]} - self.stack_depth = 0 # breaks if the depth becomes lower than this + self.stack_id = 0 # breaks when leaving/entering this frame; 0=none def __repr__(self): return 'AllBreakpoints(%r, %r, %r, %d)' % ( self.num2name, self.watchvalues, self.watchuids, - self.stack_depth) + self.stack_id) def compare(self, other): if (self.num2name == other.num2name and - self.stack_depth == other.stack_depth): + self.stack_id == other.stack_id): if self.watchvalues == other.watchvalues: return 2 # completely equal else: @@ -37,12 +37,12 @@ return 0 # different def is_empty(self): - return len(self.num2name) == 0 and self.stack_depth == 0 + return len(self.num2name) == 0 and self.stack_id == 0 def duplicate(self): a = AllBreakpoints() a.num2name.update(self.num2name) - a.stack_depth = self.stack_depth + a.stack_id = self.stack_id return a @@ -364,7 +364,7 @@ N = (max(num2name) + 1) if num2name else 0 if cmp == 0: flat = [num2name.get(n, '') for n in range(N)] - arg1 = self.all_breakpoints.stack_depth + arg1 = self.all_breakpoints.stack_id extra = '\x00'.join(flat) self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra)) self.active.expect_ready() @@ -536,8 +536,8 @@ def edit_breakpoints(self): return self.all_breakpoints - def get_stack_depth(self): - self.active.send(Message(CMD_MOREINFO)) - msg = self.active.expect(ANSWER_MOREINFO, Ellipsis) + def get_stack_id(self): + self.active.send(Message(CMD_STACKID)) + msg = self.active.expect(ANSWER_STACKID, Ellipsis) self.active.expect_ready() return msg.arg1 From pypy.commits at gmail.com Wed Jun 29 17:48:18 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 14:48:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: 'next', 'bnext', 'finish', 'bfinish' for Python Message-ID: <57744222.c7aec20a.e31c6.1fa1@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85464:3c619ff91d4b Date: 2016-06-29 23:49 +0200 http://bitbucket.org/pypy/pypy/changeset/3c619ff91d4b/ Log: 'next', 'bnext', 'finish', 'bfinish' for Python diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -64,6 +64,9 @@ return frame def enter(self, frame): + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import enter_call + enter_call(self.topframeref(), frame) frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -84,6 +87,9 @@ # be accessed also later frame_vref() jit.virtual_ref_finish(frame_vref, frame) + if self.space.config.translation.reverse_debugger: + from pypy.interpreter.reverse_debugging import leave_call + leave_call(self.topframeref(), frame) # ________________________________________________________________ diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -10,6 +10,7 @@ class DBState: extend_syntax_with_dollar_num = False + breakpoint_stack_id = 0 breakpoint_funcnames = [] printed_objects = {} metavars = [] @@ -37,8 +38,8 @@ revdb.register_debug_command(revdb.CMD_PRINT, lambda_print) revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace) revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals) - #revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints) - #revdb.register_debug_command(revdb.CMD_MOREINFO, lambda_moreinfo) + revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints) + revdb.register_debug_command(revdb.CMD_STACKID, lambda_stackid) revdb.register_debug_command("ALLOCATING", lambda_allocating) revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid) #revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch) @@ -48,6 +49,16 @@ pycode.PyCode.co_revdb_linestarts = None # or a string: an array of bits +def enter_call(caller_frame, callee_frame): + if dbstate.breakpoint_stack_id != 0: + if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): + revdb.breakpoint(-1) + +def leave_call(caller_frame, callee_frame): + if dbstate.breakpoint_stack_id != 0: + if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): + revdb.breakpoint(-1) + def potential_stop_point(frame): if not we_are_translated(): return @@ -369,6 +380,22 @@ lambda_locals = lambda: command_locals +def command_breakpoints(cmd, extra): + dbstate.breakpoint_stack_id = cmd.c_arg1 +lambda_breakpoints = lambda: command_breakpoints + +def command_stackid(cmd, extra): + frame = fetch_cur_frame() + if frame is not None and cmd.c_arg1 != 0: # parent_flag + frame = dbstate.space.getexecutioncontext().getnextframe_nohidden(frame) + if frame is None: + uid = 0 + else: + uid = revdb.get_unique_id(frame) + revdb.send_answer(revdb.ANSWER_STACKID, uid) +lambda_stackid = lambda: command_stackid + + def command_allocating(uid, gcref): w_obj = cast_gcref_to_instance(W_Root, gcref) dbstate.printed_objects[uid] = w_obj diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -154,18 +154,20 @@ self.remove_tainting() try: self.pgroup.go_forward(steps) - return True + return None except Breakpoint as b: self.hit_breakpoint(b) - return False + return b - def move_backward(self, steps): + def move_backward(self, steps, rel_stop_at=-1): + ignore_bkpt = steps == 1 and rel_stop_at == -1 try: - self.pgroup.go_backward(steps, ignore_breakpoints=(steps==1)) - return True + self.pgroup.go_backward(steps, ignore_breakpoints=ignore_bkpt, + rel_stop_at=rel_stop_at) + return None except Breakpoint as b: self.hit_breakpoint(b, backward=True) - return False + return b def hit_breakpoint(self, b, backward=False): if b.num != -1: @@ -198,7 +200,7 @@ @contextmanager def _stack_id_break(self, stack_id): # add temporarily a breakpoint that hits when we enter/leave - # the frame identified by 'stack_id' + # a frame from/to the frame identified by 'stack_id' b = self.pgroup.edit_breakpoints() b.stack_id = stack_id try: @@ -208,50 +210,55 @@ def command_next(self, argument): """Run forward for one step, skipping calls""" - depth1 = self.pgroup.get_stack_id() - with self._stack_id_break(depth1): - if not self.move_forward(1): - # we either hit a regular breakpoint, or we hit the - # temporary breakpoint - return - if depth1 == 0: # we started outside any frame - return - if self.pgroup.get_stack_id() == depth1: - return # we are still in the same frame - # - # If, after running one step, the stack id is different than - # before but we didn't leave that frame, then we must have - # entered a new one. Continue until we leave that new frame. - # Can't do it more directly because the "breakpoint" of - # stack_id is only checked for on function enters and returns - # (which simplifies and speeds up things for the RPython - # code). - self.command_finish('') + stack_id = self.pgroup.get_stack_id(is_parent=False) + with self._stack_id_break(stack_id): + b = self.move_forward(1) + if b is None: + return # no breakpoint hit, and no frame just entered: done + elif b.num != -1: + return # a regular breakpoint was hit + else: + # we entered a frame. Continue running until we leave that + # frame again + with self._stack_id_break(stack_id): + self.command_continue("") command_n = command_next def command_bnext(self, argument): """Run backward for one step, skipping calls""" - # similar to command_next() - depth1 = self.pgroup.get_stack_id() - with self._stack_id_break(depth1): - if not self.move_backward(1): - return - if depth1 == 0: - return - if self.pgroup.get_stack_id() == depth1: - return # we are still in the same frame - self.command_bfinish('') + stack_id = self.pgroup.get_stack_id(is_parent=False) + with self._stack_id_break(stack_id): + b = self.move_backward(1, rel_stop_at=0) + if b is None: + return # no breakpoint hit, and no frame just + # reverse-entered: done + elif b.num != -1: + return # a regular breakpoint was hit + else: + # we reverse-entered a frame. Continue running backward + # until we go past the reverse-leave (i.e. the entering) + # of that frame. + with self._stack_id_break(stack_id): + self.command_bcontinue("") command_bn = command_bnext def command_finish(self, argument): """Run forward until the current function finishes""" - with self._stack_id_break(self.pgroup.get_stack_id()): - self.command_continue('') + stack_id = self.pgroup.get_stack_id(is_parent=True) + if stack_id == 0: + print 'No stack.' + else: + with self._stack_id_break(stack_id): + self.command_continue('') def command_bfinish(self, argument): """Run backward until the current function is called""" - with self._stack_id_break(self.pgroup.get_stack_id()): - self.command_bcontinue('') + stack_id = self.pgroup.get_stack_id(is_parent=True) + if stack_id == 0: + print 'No stack.' + else: + with self._stack_id_break(stack_id): + self.command_bcontinue('') def command_continue(self, argument): """Run forward""" diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -58,8 +58,8 @@ # CMD_STACKID returns the id of the current or parent frame (depending # on the 'parent-flag' passed in), or 0 if not found. The id can be just # the stack depth, or it can be the unique id of the frame object. When -# used in CMD_BREAKPOINTS, it means "break if we are entering/leaving that -# frame". +# used in CMD_BREAKPOINTS, it means "break if we are entering/leaving a +# frame from/to the given frame". # Message(ANSWER_STACKID, stack-id) ANSWER_STACKID = 21 diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -19,10 +19,11 @@ self.num2name = {} # {small number: break/watchpoint} self.watchvalues = {} # {small number: resulting text} self.watchuids = {} # {small number: [uid...]} - self.stack_id = 0 # breaks when leaving/entering this frame; 0=none + self.stack_id = 0 # breaks when leaving/entering a frame from/to + # the frame identified by 'stack_id' def __repr__(self): - return 'AllBreakpoints(%r, %r, %r, %d)' % ( + return 'AllBreakpoints(%r, %r, %r, %r)' % ( self.num2name, self.watchvalues, self.watchuids, self.stack_id) @@ -304,7 +305,7 @@ if bkpt: raise bkpt - def go_backward(self, steps, ignore_breakpoints=False): + def go_backward(self, steps, ignore_breakpoints=False, rel_stop_at=-1): """Go backward, for the given number of 'steps' of time. Closes the active process. Implemented as jump_in_time() @@ -321,7 +322,7 @@ first_steps = 957 self._backward_search_forward( search_start_time = initial_time - first_steps, - search_stop_time = initial_time - 1, + search_stop_time = initial_time + rel_stop_at, search_go_on_until_time = initial_time - steps) def _backward_search_forward(self, search_start_time, search_stop_time, @@ -536,8 +537,8 @@ def edit_breakpoints(self): return self.all_breakpoints - def get_stack_id(self): - self.active.send(Message(CMD_STACKID)) + def get_stack_id(self, is_parent): + self.active.send(Message(CMD_STACKID, is_parent)) msg = self.active.expect(ANSWER_STACKID, Ellipsis) self.active.expect_ready() return msg.arg1 diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1097,9 +1097,8 @@ void rpy_reverse_db_breakpoint(int64_t num) { if (flag_io_disabled != FID_REGULAR_MODE) { - fprintf(stderr, "revdb.breakpoint(): cannot be called from a " - "debug command\n"); - exit(1); + /* called from a debug command, ignore */ + return; } switch (breakpoint_mode) { From pypy.commits at gmail.com Thu Jun 30 01:54:35 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 22:54:35 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: breakpoints Message-ID: <5774b41b.571a1c0a.93495.5b05@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85465:b4eaa8e5aef7 Date: 2016-06-30 07:53 +0200 http://bitbucket.org/pypy/pypy/changeset/b4eaa8e5aef7/ Log: breakpoints diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -11,7 +11,7 @@ class DBState: extend_syntax_with_dollar_num = False breakpoint_stack_id = 0 - breakpoint_funcnames = [] + breakpoint_funcnames = None printed_objects = {} metavars = [] watch_progs = [] @@ -31,7 +31,6 @@ dbstate.space = space dbstate.w_future = space.w_Ellipsis # a random prebuilt object - make_sure_not_resized(dbstate.breakpoint_funcnames) make_sure_not_resized(dbstate.watch_progs) make_sure_not_resized(dbstate.metavars) @@ -50,6 +49,10 @@ def enter_call(caller_frame, callee_frame): + if dbstate.breakpoint_funcnames is not None: + name = callee_frame.getcode().co_name + if name in dbstate.breakpoint_funcnames: + revdb.breakpoint(dbstate.breakpoint_funcnames[name]) if dbstate.breakpoint_stack_id != 0: if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame): revdb.breakpoint(-1) @@ -382,6 +385,23 @@ def command_breakpoints(cmd, extra): dbstate.breakpoint_stack_id = cmd.c_arg1 + funcnames = None + for i, name in enumerate(extra.split('\x00')): + if name: + if name[0] == 'B': + if funcnames is None: + funcnames = {} + funcnames[name[1:]] = i + elif name[0] == 'W': + pass + ## try: + ## prog = compiler.parse(dbstate.space, name[1:]) + ## except DuhtonError, e: + ## revdb.send_output('compiling "%s": %s\n' % + ## (name[1:], e.msg)) + ## else: + ## watch_progs.append((prog, i, '')) + dbstate.breakpoint_funcnames = funcnames lambda_breakpoints = lambda: command_breakpoints def command_stackid(cmd, extra): From pypy.commits at gmail.com Thu Jun 30 02:51:20 2016 From: pypy.commits at gmail.com (arigo) Date: Wed, 29 Jun 2016 23:51:20 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: make compiling watch points a separate phase, returning a marshalled code Message-ID: <5774c168.0ed11c0a.36e12.ffffbcaa@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85466:506d89aa4799 Date: 2016-06-30 08:52 +0200 http://bitbucket.org/pypy/pypy/changeset/506d89aa4799/ Log: make compiling watch points a separate phase, returning a marshalled code diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -16,8 +16,9 @@ CMD_BREAKPOINTS = 4 CMD_STACKID = 5 CMD_ATTACHID = 6 -CMD_CHECKWATCH = 7 -CMD_WATCHVALUES = 8 +CMD_COMPILEWATCH= 7 +CMD_CHECKWATCH = 8 +CMD_WATCHVALUES = 9 ANSWER_LINECACHE= 19 ANSWER_TEXT = 20 ANSWER_STACKID = 21 diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -120,32 +120,40 @@ lst = [str(n) for n in sorted(self.pgroup.paused)] print ', '.join(lst) - def _bp_kind(self, name): + def _bp_kind(self, num): + name = self.pgroup.all_breakpoints.num2name.get(num, '??') if name[0] == 'B': - return 'breakpoint' + kind = 'breakpoint' + name = name[1:] elif name[0] == 'W': - return 'watchpoint' + kind = 'watchpoint' + name = self.pgroup.all_breakpoints.sources.get(num, '??') else: - return '?????point' + kind = '?????point' + name = repr(name) + return kind, name - def _bp_new(self, break_at, nids=None): + def _bp_new(self, break_at, nids=None, source_expr=None): b = self.pgroup.edit_breakpoints() new = 1 while new in b.num2name: new += 1 b.num2name[new] = break_at + b.sources[new] = source_expr if break_at.startswith('W'): b.watchvalues[new] = '' if nids: b.watchuids[new] = self.pgroup.nids_to_uids(nids) - print "%s %d added" % (self._bp_kind(break_at).capitalize(), new) + kind, name = self._bp_kind(new) + print "%s %d added" % (kind.capitalize(), new) def cmd_info_breakpoints(self): """List current breakpoints and watchpoints""" - lst = self.pgroup.all_breakpoints.num2name.items() + lst = self.pgroup.all_breakpoints.num2name.keys() if lst: - for num, name in sorted(lst): - print '\t%s %d: %s' % (self._bp_kind(name), num, name[1:]) + for num in sorted(lst): + kind, name = self._bp_kind(num) + print '\t%s %d: %s' % (kind, num, name) else: print 'no breakpoints.' cmd_info_watchpoints = cmd_info_breakpoints @@ -171,10 +179,9 @@ def hit_breakpoint(self, b, backward=False): if b.num != -1: - name = self.pgroup.all_breakpoints.num2name.get(b.num, '??') - kind = self._bp_kind(name) + kind, name = self._bp_kind(d.num) self.print_extra_pending_info = 'Hit %s %d: %s' % (kind, b.num, - name[1:]) + name) elif backward: b.time -= 1 if self.pgroup.get_current_time() != b.time: @@ -307,22 +314,31 @@ if arg not in b.num2name: print "No breakpoint/watchpoint number %d" % (arg,) else: - name = b.num2name.pop(arg) + kind, name = self._bp_kind(arg) + b.num2name.pop(arg, '') + b.sources.pop(arg, '') b.watchvalues.pop(arg, '') b.watchuids.pop(arg, '') - kind = self._bp_kind(name) - print "%s %d deleted: %s" % (kind.capitalize(), arg, name[1:]) + print "%s %d deleted: %s" % (kind.capitalize(), arg, name) def command_watch(self, argument): """Add a watchpoint (use $NUM in the expression to watch)""" if not argument: print "Watch what?" return + # + ok_flag, compiled_code = self.pgroup.compile_watchpoint_expr(argument) + if not ok_flag: + print compiled_code # the error message + print 'Watchpoint not added' + return + # nids = map(int, r_dollar_num.findall(argument)) - ok_flag, text = self.pgroup.check_watchpoint_expr(argument, nids) + ok_flag, text = self.pgroup.check_watchpoint_expr(compiled_code, nids) if not ok_flag: print text print 'Watchpoint not added' - else: - self._bp_new('W' + argument, nids=nids) - self.pgroup.update_watch_values() + return + # + self._bp_new('W' + compiled_code, nids=nids, source_expr=argument) + self.pgroup.update_watch_values() diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py --- a/rpython/translator/revdb/message.py +++ b/rpython/translator/revdb/message.py @@ -18,8 +18,9 @@ # extra="\0-separated names") CMD_STACKID = 5 # Message(CMD_STACKID, parent-flag) CMD_ATTACHID = 6 # Message(CMD_ATTACHID, small-num, unique-id) -CMD_CHECKWATCH = 7 # Message(CMD_CHECKWATCH, extra=expression) -CMD_WATCHVALUES = 8 # Message(CMD_WATCHVALUES, extra=texts) +CMD_COMPILEWATCH= 7 # Message(CMD_COMPILEWATCH, extra=expression) +CMD_CHECKWATCH = 8 # Message(CMD_CHECKWATCH, extra=compiled_code) +CMD_WATCHVALUES = 9 # Message(CMD_WATCHVALUES, extra=texts) # the first message sent by the first child: @@ -67,6 +68,8 @@ # Message(ANSWER_NEXTNID, unique-id) ANSWER_NEXTNID = 22 +# sent after CMD_COMPILEWATCH: +# Message(ANSWER_WATCH, ok_flag, extra=marshalled_code) # sent after CMD_CHECKWATCH: # Message(ANSWER_WATCH, ok_flag, extra=result_of_expr) ANSWER_WATCH = 23 diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -17,6 +17,7 @@ def __init__(self): self.num2name = {} # {small number: break/watchpoint} + self.sources = {} # {small number: src text or None} self.watchvalues = {} # {small number: resulting text} self.watchuids = {} # {small number: [uid...]} self.stack_id = 0 # breaks when leaving/entering a frame from/to @@ -402,11 +403,17 @@ seen.add(num) assert set(self.all_breakpoints.watchvalues) == seen - def check_watchpoint_expr(self, expr, nids=None): + def compile_watchpoint_expr(self, expr): + self.active.send(Message(CMD_COMPILEWATCH, extra=expr)) + msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis) + self.active.expect_ready() + return msg.arg1, msg.extra + + def check_watchpoint_expr(self, compiled_code, nids=None): if nids: uids = self.nids_to_uids(nids) self.attach_printed_objects(uids, watch_env=True) - self.active.send(Message(CMD_CHECKWATCH, extra=expr)) + self.active.send(Message(CMD_CHECKWATCH, extra=compiled_code)) msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis) self.active.expect_ready() return msg.arg1, msg.extra From pypy.commits at gmail.com Thu Jun 30 04:14:23 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 30 Jun 2016 01:14:23 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Watchpoints. Still buggy Message-ID: <5774d4df.9a4a1c0a.170bb.ffffaa82@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85467:716b1a95fd64 Date: 2016-06-30 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/716b1a95fd64/ Log: Watchpoints. Still buggy diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -6,6 +6,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter import gateway, typedef, pycode, pytraceback +from pypy.module.marshal import interp_marshal class DBState: @@ -41,8 +42,9 @@ revdb.register_debug_command(revdb.CMD_STACKID, lambda_stackid) revdb.register_debug_command("ALLOCATING", lambda_allocating) revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid) - #revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch) - #revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) + revdb.register_debug_command(revdb.CMD_COMPILEWATCH, lambda_compilewatch) + revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch) + revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues) pycode.PyCode.co_revdb_linestarts = None # or a string: an array of bits @@ -138,13 +140,13 @@ def stop_point_at_start_of_line(): if revdb.watch_save_state(): any_watch_point = False - #for prog, watch_id, expected in dbstate.watch_progs: - # any_watch_point = True - # got = _watch_expr(prog) - # if got != expected: - # break - #else: - watch_id = -1 + for prog, watch_id, expected in dbstate.watch_progs: + any_watch_point = True + got = _run_watch(prog) + if got != expected: + break + else: + watch_id = -1 revdb.watch_restore_state(any_watch_point) if watch_id != -1: revdb.breakpoint(watch_id) @@ -274,6 +276,7 @@ break show_frame(tb.frame, tb.get_lineno(), indent=' ') tb = tb.next + revdb.send_output('%s\n' % operationerr.errorstr(space)) # set the sys.last_xxx attributes w_type = operationerr.w_type @@ -283,9 +286,6 @@ space.setitem(space.sys.w_dict, space.wrap('last_value'), w_value) space.setitem(space.sys.w_dict, space.wrap('last_traceback'), w_tb) - # re-raise, catch me in the outside "except OperationError" - raise - except OperationError as e: revdb.send_output('%s\n' % e.errorstr(space, use_repr=True)) lambda_print = lambda: command_print @@ -384,26 +384,33 @@ def command_breakpoints(cmd, extra): + space = dbstate.space dbstate.breakpoint_stack_id = cmd.c_arg1 funcnames = None - for i, name in enumerate(extra.split('\x00')): - if name: - if name[0] == 'B': - if funcnames is None: - funcnames = {} - funcnames[name[1:]] = i - elif name[0] == 'W': - pass - ## try: - ## prog = compiler.parse(dbstate.space, name[1:]) - ## except DuhtonError, e: - ## revdb.send_output('compiling "%s": %s\n' % - ## (name[1:], e.msg)) - ## else: - ## watch_progs.append((prog, i, '')) + watch_progs = [] + for i, kind, name in revdb.split_breakpoints_arg(extra): + if kind == 'B': + if funcnames is None: + funcnames = {} + funcnames[name] = i + elif kind == 'W': + code = interp_marshal.loads(space, space.wrap(name)) + watch_progs.append((code, i, '')) dbstate.breakpoint_funcnames = funcnames + dbstate.watch_progs = watch_progs[:] lambda_breakpoints = lambda: command_breakpoints + +def command_watchvalues(cmd, extra): + expected = extra.split('\x00') + for j in range(len(dbstate.watch_progs)): + prog, i, _ = dbstate.watch_progs[j] + if i >= len(expected): + raise IndexError + dbstate.watch_progs[j] = prog, i, expected[i] +lambda_watchvalues = lambda: command_watchvalues + + def command_stackid(cmd, extra): frame = fetch_cur_frame() if frame is not None and cmd.c_arg1 != 0: # parent_flag @@ -440,3 +447,38 @@ w_obj = dbstate.w_future set_metavar(index_metavar, w_obj) lambda_attachid = lambda: command_attachid + + +def command_compilewatch(cmd, expression): + space = dbstate.space + try: + code = compile(expression, 'eval') + marshalled_code = space.str_w(interp_marshal.dumps( + space, space.wrap(code), + space.wrap(interp_marshal.Py_MARSHAL_VERSION))) + except OperationError as e: + revdb.send_watch(e.errorstr(space), ok_flag=0) + else: + revdb.send_watch(marshalled_code, ok_flag=1) +lambda_compilewatch = lambda: command_compilewatch + +def command_checkwatch(cmd, marshalled_code): + space = dbstate.space + try: + code = interp_marshal.loads(space, space.wrap(marshalled_code)) + w_res = code.exec_code(space, space.builtin, space.builtin) + text = space.str_w(space.repr(w_res)) + except OperationError as e: + revdb.send_watch(e.errorstr(space), ok_flag=0) + else: + revdb.send_watch(text, ok_flag=1) +lambda_checkwatch = lambda: command_checkwatch + +def _run_watch(code): + space = dbstate.space + try: + w_res = code.exec_code(space, space.builtin, space.builtin) + text = space.str_w(space.repr(w_res)) + except OperationError as e: + return e.errorstr(space) + return text diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py --- a/rpython/rlib/revdb.py +++ b/rpython/rlib/revdb.py @@ -118,6 +118,25 @@ llop.revdb_watch_restore_state(lltype.Void, any_watch_point) +def split_breakpoints_arg(breakpoints): + # RPython generator to help in splitting the string arg in CMD_BREAKPOINTS + n = 0 + i = 0 + while i < len(breakpoints): + kind = breakpoints[i] + i += 1 + if kind != '\x00': + length = (ord(breakpoints[i]) | + (ord(breakpoints[i + 1]) << 8) | + (ord(breakpoints[i + 2]) << 16)) + assert length >= 0 + i += 3 + yield n, kind, breakpoints[i : i + length] + i += length + n += 1 + assert i == len(breakpoints) + + # ____________________________________________________________ diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py --- a/rpython/translator/revdb/interact.py +++ b/rpython/translator/revdb/interact.py @@ -121,26 +121,32 @@ print ', '.join(lst) def _bp_kind(self, num): - name = self.pgroup.all_breakpoints.num2name.get(num, '??') - if name[0] == 'B': + break_at = self.pgroup.all_breakpoints.num2break.get(num, '??') + if break_at[0] == 'B': kind = 'breakpoint' - name = name[1:] - elif name[0] == 'W': + name = break_at[4:] + elif break_at[0] == 'W': kind = 'watchpoint' name = self.pgroup.all_breakpoints.sources.get(num, '??') else: kind = '?????point' - name = repr(name) + name = repr(break_at) return kind, name - def _bp_new(self, break_at, nids=None, source_expr=None): + def _bp_new(self, source_expr, break_code, break_at, nids=None): b = self.pgroup.edit_breakpoints() new = 1 - while new in b.num2name: + while new in b.num2break: new += 1 - b.num2name[new] = break_at + if len(break_at) > 0xFFFFFF: + raise OverflowError("break/watchpoint too complex") + b.num2break[new] = (break_code + + chr(len(break_at) & 0xFF) + + chr((len(break_at) >> 8) & 0xFF) + + chr(len(break_at) >> 16) + + break_at) b.sources[new] = source_expr - if break_at.startswith('W'): + if break_code == 'W': b.watchvalues[new] = '' if nids: b.watchuids[new] = self.pgroup.nids_to_uids(nids) @@ -149,7 +155,7 @@ def cmd_info_breakpoints(self): """List current breakpoints and watchpoints""" - lst = self.pgroup.all_breakpoints.num2name.keys() + lst = self.pgroup.all_breakpoints.num2break.keys() if lst: for num in sorted(lst): kind, name = self._bp_kind(num) @@ -179,7 +185,7 @@ def hit_breakpoint(self, b, backward=False): if b.num != -1: - kind, name = self._bp_kind(d.num) + kind, name = self._bp_kind(b.num) self.print_extra_pending_info = 'Hit %s %d: %s' % (kind, b.num, name) elif backward: @@ -304,18 +310,18 @@ if not argument: print "Break where?" return - self._bp_new('B' + argument) + self._bp_new(argument, 'B', argument) command_b = command_break def command_delete(self, argument): """Delete a breakpoint/watchpoint""" arg = int(argument) b = self.pgroup.edit_breakpoints() - if arg not in b.num2name: + if arg not in b.num2break: print "No breakpoint/watchpoint number %d" % (arg,) else: kind, name = self._bp_kind(arg) - b.num2name.pop(arg, '') + b.num2break.pop(arg, '') b.sources.pop(arg, '') b.watchvalues.pop(arg, '') b.watchuids.pop(arg, '') @@ -340,5 +346,5 @@ print 'Watchpoint not added' return # - self._bp_new('W' + compiled_code, nids=nids, source_expr=argument) + self._bp_new(argument, 'W', compiled_code, nids=nids) self.pgroup.update_watch_values() diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py --- a/rpython/translator/revdb/process.py +++ b/rpython/translator/revdb/process.py @@ -16,8 +16,8 @@ class AllBreakpoints(object): def __init__(self): - self.num2name = {} # {small number: break/watchpoint} - self.sources = {} # {small number: src text or None} + self.num2break = {} # {small number: encoded break/watchpoint} + self.sources = {} # {small number: src text} self.watchvalues = {} # {small number: resulting text} self.watchuids = {} # {small number: [uid...]} self.stack_id = 0 # breaks when leaving/entering a frame from/to @@ -25,11 +25,11 @@ def __repr__(self): return 'AllBreakpoints(%r, %r, %r, %r)' % ( - self.num2name, self.watchvalues, self.watchuids, + self.num2break, self.watchvalues, self.watchuids, self.stack_id) def compare(self, other): - if (self.num2name == other.num2name and + if (self.num2break == other.num2break and self.stack_id == other.stack_id): if self.watchvalues == other.watchvalues: return 2 # completely equal @@ -39,11 +39,11 @@ return 0 # different def is_empty(self): - return len(self.num2name) == 0 and self.stack_id == 0 + return len(self.num2break) == 0 and self.stack_id == 0 def duplicate(self): a = AllBreakpoints() - a.num2name.update(self.num2name) + a.num2break.update(self.num2break) a.stack_id = self.stack_id return a @@ -362,24 +362,24 @@ # update the breakpoints/watchpoints self.active.breakpoints_cache = None - num2name = self.all_breakpoints.num2name - N = (max(num2name) + 1) if num2name else 0 + num2break = self.all_breakpoints.num2break + N = (max(num2break) + 1) if num2break else 0 if cmp == 0: - flat = [num2name.get(n, '') for n in range(N)] + flat = [num2break.get(n, '\x00') for n in range(N)] arg1 = self.all_breakpoints.stack_id - extra = '\x00'.join(flat) + extra = ''.join(flat) self.active.send(Message(CMD_BREAKPOINTS, arg1, extra=extra)) self.active.expect_ready() else: assert cmp == 1 # update the watchpoint values - if any(name.startswith('W') for name in num2name.values()): + if any(name.startswith('W') for name in num2break.values()): watchvalues = self.all_breakpoints.watchvalues flat = [] for n in range(N): text = '' - name = num2name.get(n, '') + name = num2break.get(n, '') if name.startswith('W'): text = watchvalues[n] flat.append(text) @@ -392,13 +392,13 @@ def update_watch_values(self): self._update_watchpoints_uids() seen = set() - for num, name in self.all_breakpoints.num2name.items(): + for num, name in self.all_breakpoints.num2break.items(): if name.startswith('W'): - _, text = self.check_watchpoint_expr(name[1:]) + _, text = self.check_watchpoint_expr(name[4:]) if text != self.all_breakpoints.watchvalues[num]: #print self.active.pid print 'updating watchpoint value: %s => %s' % ( - name[1:], text) + self.all_breakpoints.sources[num], text) self.all_breakpoints.watchvalues[num] = text seen.add(num) assert set(self.all_breakpoints.watchvalues) == seen From pypy.commits at gmail.com Thu Jun 30 06:35:12 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 30 Jun 2016 03:35:12 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: bug fixes, there is more Message-ID: <5774f5e0.c255c20a.b048d.ffffacc4@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85468:550befa14f60 Date: 2016-06-30 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/550befa14f60/ Log: bug fixes, there is more diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py --- a/pypy/interpreter/reverse_debugging.py +++ b/pypy/interpreter/reverse_debugging.py @@ -140,10 +140,13 @@ def stop_point_at_start_of_line(): if revdb.watch_save_state(): any_watch_point = False + space = dbstate.space for prog, watch_id, expected in dbstate.watch_progs: any_watch_point = True - got = _run_watch(prog) - if got != expected: + try: + if _run_watch(space, prog) != expected: + break + except OperationError: break else: watch_id = -1 @@ -466,19 +469,15 @@ space = dbstate.space try: code = interp_marshal.loads(space, space.wrap(marshalled_code)) - w_res = code.exec_code(space, space.builtin, space.builtin) - text = space.str_w(space.repr(w_res)) + text = _run_watch(space, code) except OperationError as e: revdb.send_watch(e.errorstr(space), ok_flag=0) else: revdb.send_watch(text, ok_flag=1) lambda_checkwatch = lambda: command_checkwatch -def _run_watch(code): - space = dbstate.space - try: - w_res = code.exec_code(space, space.builtin, space.builtin) - text = space.str_w(space.repr(w_res)) - except OperationError as e: - return e.errorstr(space) - return text + +def _run_watch(space, prog): + w_dict = space.builtin.w_dict + w_res = prog.exec_code(space, w_dict, w_dict) + return space.str_w(space.repr(w_res)) diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -29,7 +29,7 @@ ll_malloc_varsize = mh.ll_malloc_varsize fields = [("hash", lltype.Signed)] - if translator.config.translation.reverse_debugger: + if translator and translator.config.translation.reverse_debugger: fields.append(("uid", lltype.SignedLongLong)) self.HDR = lltype.Struct("header", *fields) HDRPTR = lltype.Ptr(self.HDR) @@ -74,11 +74,21 @@ # XXX same behavior for zero=True: in theory that's wrong if TYPE._is_atomic(): funcptr = self.malloc_fixedsize_atomic_ptr + opname = 'boehm_malloc_atomic' else: funcptr = self.malloc_fixedsize_ptr - v_raw = hop.genop("direct_call", - [funcptr, c_size], - resulttype=llmemory.GCREF) + opname = 'boehm_malloc' + tr = self.translator + if tr and tr.config.translation.reverse_debugger: + # Don't check for NULLs after the operation (it crashes anyway + # with an explicit error message in case of out-of-memory). + # Avoiding a direct_call lets _RPY_REVDB_PRUID() prints the + # right file/line, at least for fixed-size mallocs. + v_raw = hop.genop(opname, [c_size], resulttype=llmemory.GCREF) + else: + v_raw = hop.genop("direct_call", + [funcptr, c_size], + resulttype=llmemory.GCREF) finalizer_ptr = self.finalizer_funcptr_for_type(TYPE) if finalizer_ptr: c_finalizer_ptr = Constant(finalizer_ptr, self.FINALIZER_PTR) diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -279,20 +279,13 @@ { /* Boehm only */ if (obj->h_hash == 0) { - Signed h; - if (flag_io_disabled != FID_REGULAR_MODE) { - /* This is when running debug commands. Don't cache the - hash on the object at all. */ - return ~((Signed)obj); - } - /* When recording, we get the hash the normal way from the - pointer casted to an int, and record that. When replaying, - we read it from the record. In both cases, we cache the - hash in the object, so that we record/replay only once per - object. */ - RPY_REVDB_EMIT(h = ~((Signed)obj);, Signed _e, h); - assert(h != 0); - obj->h_hash = h; + /* We never need to record anything: if h_hash is zero (which + is the case for all newly allocated objects), then we just + copy h_uid. This gives a stable answer. This would give + 0 for all prebuilt objects, but these should not have a + null h_hash anyway. + */ + obj->h_hash = obj->h_uid; } return obj->h_hash; } @@ -853,6 +846,13 @@ static void check_at_end(uint64_t stop_points) { char dummy[1]; + if (rpy_revdb.buf_p != rpy_revdb.buf_limit - 1 || + read(rpy_rev_fileno, dummy, 1) > 0) { + fprintf(stderr, "RevDB file error: corrupted file (too much data or, " + "more likely, non-deterministic run, e.g. a " + "watchpoint with side effects)\n"); + exit(1); + } if (stop_points != rpy_revdb.stop_point_seen) { fprintf(stderr, "Bad number of stop points " "(seen %lld, recorded %lld)\n", @@ -860,11 +860,6 @@ (long long)stop_points); exit(1); } - if (rpy_revdb.buf_p != rpy_revdb.buf_limit - 1 || - read(rpy_rev_fileno, dummy, 1) > 0) { - fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n"); - exit(1); - } if (stop_points != total_stop_points) { fprintf(stderr, "RevDB file modified while reading?\n"); exit(1); @@ -971,6 +966,7 @@ memcpy(future_ids, extra, cmd->extra_size); future_ids[cmd->extra_size / sizeof(uint64_t)] = 0; uid_break = *future_ids; + attach_gdb(); } future_next_id = future_ids; } @@ -1168,10 +1164,17 @@ return uid; } +struct destructor_s { + void *d_obj; + void (*d_callback)(void *); +}; + static int _ftree_compare(const void *obj1, const void *obj2) { - const struct pypy_header0 *h1 = obj1; - const struct pypy_header0 *h2 = obj2; + const struct destructor_s *d1 = obj1; + const struct destructor_s *d2 = obj2; + struct pypy_header0 *h1 = d1->d_obj; + struct pypy_header0 *h2 = d2->d_obj; if (h1->h_uid < h2->h_uid) return -1; if (h1->h_uid == h2->h_uid) @@ -1180,26 +1183,60 @@ return 1; } +static void _ftree_add(void **tree, void *obj, void (*callback)(void *)) +{ + /* Note: we always allocate an indirection through a + struct destructor_s, so that Boehm knows that 'obj' must be + kept alive. */ + struct destructor_s *node, **item; + node = GC_MALLOC_UNCOLLECTABLE(sizeof(struct destructor_s)); + node->d_obj = obj; + node->d_callback = callback; + item = tsearch(node, tree, _ftree_compare); + if (item == NULL) { + fprintf(stderr, "_ftree_add: out of memory\n"); + exit(1); + } + if (*item != node) { + fprintf(stderr, "_ftree_add: duplicate object\n"); + exit(1); + } +} + +static struct pypy_header0 *_ftree_pop(void **tree, uint64_t uid, + void (**callback_out)(void *)) +{ + struct destructor_s d_dummy, *entry, **item; + struct pypy_header0 o_dummy, *result; + + d_dummy.d_obj = &o_dummy; + o_dummy.h_uid = uid; + item = tfind(&d_dummy, tree, _ftree_compare); + if (item == NULL) { + fprintf(stderr, "_ftree_pop: object not found\n"); + exit(1); + } + entry = *item; + result = entry->d_obj; + if (callback_out) + *callback_out = entry->d_callback; + assert(result->h_uid == uid); + tdelete(entry, tree, _ftree_compare); + GC_FREE(entry); + return result; +} + RPY_EXTERN int rpy_reverse_db_fq_register(void *obj) { - /*fprintf(stderr, "FINALIZER_TREE: %lld -> %p\n", + fprintf(stderr, "FINALIZER_TREE: %lld -> %p\n", ((struct pypy_header0 *)obj)->h_uid, obj); - */ if (!RPY_RDB_REPLAY) { return 0; /* recording */ } else { - /* add the object into the finalizer_tree, keyed by the h_uid */ - void **item = tsearch(obj, &finalizer_tree, _ftree_compare); - if (item == NULL) { - fprintf(stderr, "fq_register: out of memory\n"); - exit(1); - } - if (*item != obj) { - fprintf(stderr, "fq_register: duplicate object\n"); - exit(1); - } + /* add the object into the finalizer_tree, keyed by the h_uid. */ + _ftree_add(&finalizer_tree, obj, NULL); return 1; /* replaying */ } } @@ -1210,47 +1247,19 @@ int64_t uid; RPY_REVDB_EMIT(uid = result ? ((struct pypy_header0 *)result)->h_uid : -1;, int64_t _e, uid); + fprintf(stderr, "next_dead: object %lld\n", uid); if (RPY_RDB_REPLAY) { if (uid == -1) { result = NULL; } else { /* fetch and remove the object from the finalizer_tree */ - void **item; - struct pypy_header0 dummy; - dummy.h_uid = uid; - item = tfind(&dummy, &finalizer_tree, _ftree_compare); - if (item == NULL) { - fprintf(stderr, "next_dead: object not found\n"); - exit(1); - } - result = *item; - assert(((struct pypy_header0 *)result)->h_uid == uid); - tdelete(result, &finalizer_tree, _ftree_compare); + result = _ftree_pop(&finalizer_tree, uid, NULL); } } return result; } -struct destructor_s { - void *obj; - void (*callback)(void *); -}; - - static int _dtree_compare(const void *obj1, const void *obj2) -{ - const struct destructor_s *d1 = obj1; - const struct destructor_s *d2 = obj2; - const struct pypy_header0 *h1 = d1->obj; - const struct pypy_header0 *h2 = d2->obj; - if (h1->h_uid < h2->h_uid) - return -1; - if (h1->h_uid == h2->h_uid) - return 0; - else - return 1; -} - RPY_EXTERN void rpy_reverse_db_register_destructor(void *obj, void (*callback)(void *)) { @@ -1259,23 +1268,7 @@ NULL, NULL, NULL); } else { - struct destructor_s **item; - struct destructor_s *node = malloc(sizeof(struct destructor_s)); - if (!node) { - fprintf(stderr, "register_destructor: malloc: out of memory\n"); - exit(1); - } - node->obj = obj; - node->callback = callback; - item = tsearch(node, &destructor_tree, _dtree_compare); - if (item == NULL) { - fprintf(stderr, "register_destructor: tsearch: out of memory\n"); - exit(1); - } - if (*item != node) { - fprintf(stderr, "register_destructor: duplicate object\n"); - exit(1); - } + _ftree_add(&destructor_tree, obj, callback); } } @@ -1292,26 +1285,15 @@ while (1) { int64_t uid; - struct destructor_s d_dummy, *entry, **item; - struct pypy_header0 o_dummy; + struct pypy_header0 *obj; + void (*callback)(void *); RPY_REVDB_EMIT(abort();, int64_t _e, uid); if (uid == -1) break; - d_dummy.obj = &o_dummy; - o_dummy.h_uid = uid; - item = tfind(&d_dummy, &destructor_tree, _dtree_compare); - if (item == NULL) { - fprintf(stderr, "replay_call_destructors: object not found\n"); - exit(1); - } - entry = *item; - assert(((struct pypy_header0 *)entry->obj)->h_uid == uid); - tdelete(entry, &destructor_tree, _dtree_compare); - - entry->callback(entry->obj); - free(entry); + obj = _ftree_pop(&destructor_tree, uid, &callback); + callback(obj); } } diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -35,12 +35,19 @@ "%s:%d: %0*llx\n", \ __FILE__, __LINE__, 2 * sizeof(_e), \ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) +#endif + +#if 1 /* enable to print all allocs to stderr */ # define _RPY_REVDB_PRUID() \ fprintf(stderr, \ "%s:%d: obj %llu\n", \ __FILE__, __LINE__, (unsigned long long) uid) -#else +#endif + +#ifndef _RPY_REVDB_PRINT # define _RPY_REVDB_PRINT(mode) /* nothing */ +#endif +#ifndef _RPY_REVDB_PRUID # define _RPY_REVDB_PRUID() /* nothing */ #endif diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -150,8 +150,7 @@ assert match hash_value = int(match.group(1)) rdb = self.fetch_rdb([self.exename, 'Xx']) - # compute_identity_hash() call, but only the first one - x = rdb.next(); assert intmask(x) == intmask(hash_value) + # compute_identity_hash() doesn't record anything # write() call x = rdb.next(); assert x == len(out) x = rdb.next('i'); assert x == 0 # errno From pypy.commits at gmail.com Thu Jun 30 08:10:18 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 30 Jun 2016 05:10:18 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Go through the list of all llops and mark the ones that are unsafe. Message-ID: <57750c2a.e655c20a.bebbc.097c@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85469:567dcdd8e1b6 Date: 2016-06-30 14:11 +0200 http://bitbucket.org/pypy/pypy/changeset/567dcdd8e1b6/ Log: Go through the list of all llops and mark the ones that are unsafe. diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -16,11 +16,11 @@ super(BoehmGCTransformer, self).__init__(translator, inline=inline) self.finalizer_funcptrs = {} - atomic_mh = mallocHelpers() + atomic_mh = mallocHelpers(gckind='gc') atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.GCREF, size) ll_malloc_fixedsize_atomic = atomic_mh._ll_malloc_fixedsize - mh = mallocHelpers() + mh = mallocHelpers(gckind='gc') mh.allocate = lambda size: llop.boehm_malloc(llmemory.GCREF, size) ll_malloc_fixedsize = mh._ll_malloc_fixedsize diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py --- a/rpython/memory/gctransform/refcounting.py +++ b/rpython/memory/gctransform/refcounting.py @@ -54,7 +54,7 @@ def ll_no_pointer_dealloc(adr): llop.gc_free(lltype.Void, adr) - mh = mallocHelpers() + mh = mallocHelpers(gckind='gc') mh.allocate = llmemory.raw_malloc def ll_malloc_fixedsize(size): size = gc_header_offset + size diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -411,7 +411,7 @@ # ________________________________________________________________ -def mallocHelpers(): +def mallocHelpers(gckind): class _MallocHelpers(object): def _freeze_(self): return True @@ -442,9 +442,16 @@ mh._ll_malloc_varsize_no_length = _ll_malloc_varsize_no_length mh.ll_malloc_varsize_no_length = _ll_malloc_varsize_no_length + if gckind == 'raw': + llopstore = llop.raw_store + elif gckind == 'gc': + llopstore = llop.gc_store + else: + raise AssertionError(gckind) + def ll_malloc_varsize(length, size, itemsize, lengthoffset): result = mh.ll_malloc_varsize_no_length(length, size, itemsize) - llop.raw_store(lltype.Void, result, lengthoffset, length) + llopstore(lltype.Void, result, lengthoffset, length) return result mh.ll_malloc_varsize = ll_malloc_varsize @@ -464,7 +471,7 @@ def __init__(self, translator, inline=False): super(GCTransformer, self).__init__(translator, inline=inline) - mh = mallocHelpers() + mh = mallocHelpers(gckind='raw') mh.allocate = llmemory.raw_malloc ll_raw_malloc_fixedsize = mh._ll_malloc_fixedsize ll_raw_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1128,9 +1128,26 @@ def op_revdb_stop_point(self, *args): pass - def op_revdb_send_output(self, ll_string): - from rpython.rtyper.annlowlevel import hlstr - sys.stdout.write(hlstr(ll_string)) + def op_revdb_send_answer(self, *args): + raise NotImplementedError + def op_revdb_breakpoint(self, *args): + raise NotImplementedError + def op_revdb_get_value(self, *args): + raise NotImplementedError + def op_revdb_get_unique_id(self, *args): + raise NotImplementedError + + def op_revdb_watch_save_state(self, *args): + return False + + def op_revdb_watch_restore_state(self, *args): + raise NotImplementedError + def op_revdb_weakref_create(self, *args): + raise NotImplementedError + def op_revdb_weakref_deref(self, *args): + raise NotImplementedError + def op_revdb_call_destructor(self, *args): + raise NotImplementedError class Tracer(object): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -8,7 +8,8 @@ class LLOp(object): def __init__(self, sideeffects=True, canfold=False, canraise=(), - canmallocgc=False, canrun=False, tryfold=False): + canmallocgc=False, canrun=False, tryfold=False, + revdb_protect=False): # self.opname = ... (set afterwards) if canfold: @@ -41,6 +42,9 @@ # The operation can be run directly with __call__ self.canrun = canrun or canfold + # RevDB: the operation must always be protected with RPY_REVDB_EMIT() + self.revdb_protect = revdb_protect + # __________ make the LLOp instances callable from LL helpers __________ __name__ = property(lambda self: 'llop_'+self.opname) @@ -385,17 +389,19 @@ 'boehm_malloc_atomic': LLOp(), 'boehm_register_finalizer': LLOp(), 'boehm_disappearing_link': LLOp(), - 'raw_malloc': LLOp(), + 'raw_malloc': LLOp(revdb_protect=True), 'raw_malloc_usage': LLOp(sideeffects=False), - 'raw_free': LLOp(), - 'raw_memclear': LLOp(), - 'raw_memset': LLOp(), - 'raw_memcopy': LLOp(), - 'raw_memmove': LLOp(), - 'raw_load': LLOp(sideeffects=False, canrun=True), - 'raw_store': LLOp(canrun=True), - 'bare_raw_store': LLOp(), + 'raw_free': LLOp(revdb_protect=True), + 'raw_memclear': LLOp(revdb_protect=True), + 'raw_memset': LLOp(revdb_protect=True), + 'raw_memcopy': LLOp(revdb_protect=True), + 'raw_memmove': LLOp(revdb_protect=True), + 'raw_load': LLOp(revdb_protect=True, sideeffects=False, + canrun=True), + 'raw_store': LLOp(revdb_protect=True, canrun=True), + 'bare_raw_store': LLOp(revdb_protect=True), 'gc_load_indexed': LLOp(sideeffects=False, canrun=True), + 'gc_store': LLOp(canrun=True), 'stack_malloc': LLOp(), # mmh 'track_alloc_start': LLOp(), 'track_alloc_stop': LLOp(), @@ -442,7 +448,7 @@ 'get_write_barrier_failing_case': LLOp(sideeffects=False), 'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), - 'll_read_timestamp': LLOp(canrun=True), + 'll_read_timestamp': LLOp(revdb_protect=True, canrun=True), # __________ GC operations __________ @@ -456,8 +462,8 @@ # see rlib/objectmodel for gc_identityhash and gc_id 'gc_identityhash': LLOp(sideeffects=False, canmallocgc=True), 'gc_id': LLOp(sideeffects=False, canmallocgc=True), - 'gc_obtain_free_space': LLOp(), - 'gc_set_max_heap_size': LLOp(), + 'gc_obtain_free_space': LLOp(revdb_protect=True), + 'gc_set_max_heap_size': LLOp(revdb_protect=True), 'gc_can_move' : LLOp(sideeffects=False), 'gc_thread_run' : LLOp(), 'gc_thread_start' : LLOp(), @@ -523,7 +529,7 @@ # __________ misc operations __________ - 'stack_current': LLOp(sideeffects=False), + 'stack_current': LLOp(revdb_protect=True, sideeffects=False), 'keepalive': LLOp(), 'same_as': LLOp(canfold=True), 'hint': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -706,6 +706,7 @@ TVAL = lltype.typeOf(newvalue) p = rffi.cast(rffi.CArrayPtr(TVAL), p + ofs) p[0] = newvalue +op_gc_store = op_raw_store def op_raw_load(TVAL, p, ofs): from rpython.rtyper.lltypesystem import rffi diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -266,7 +266,8 @@ def gen_op(self, op): macro = 'OP_%s' % op.opname.upper() line = None - if op.opname.startswith('gc_') and op.opname != 'gc_load_indexed': + if (op.opname.startswith('gc_') and op.opname != 'gc_load_indexed' + and op.opname != 'gc_store'): meth = getattr(self.gcpolicy, macro, None) if meth: line = meth(self, op) @@ -278,6 +279,11 @@ lst = [self.expr(v) for v in op.args] lst.append(self.expr(op.result)) line = '%s(%s);' % (macro, ', '.join(lst)) + if self.db.reverse_debugger: + from rpython.translator.revdb import gencsupp + if op.opname in gencsupp.set_revdb_protected: + line = gencsupp.emit(line, self.lltypename(op.result), + self.expr(op.result)) if "\n" not in line: yield line else: @@ -435,7 +441,7 @@ return 'abort(); /* jit_conditional_call */' # low-level operations - def generic_get(self, op, sourceexpr): + def generic_get(self, op, sourceexpr, accessing_mem=True): T = self.lltypemap(op.result) newvalue = self.expr(op.result, special_case_void=False) result = '%s = %s;' % (newvalue, sourceexpr) @@ -445,7 +451,8 @@ S = self.lltypemap(op.args[0]).TO if (S._gckind != 'gc' and not S._hints.get('is_excdata') and not S._hints.get('static_immutable') - and not S._hints.get('ignore_revdb')): + and not S._hints.get('ignore_revdb') + and accessing_mem): from rpython.translator.revdb import gencsupp result = gencsupp.emit(result, self.lltypename(op.result), newvalue) @@ -464,7 +471,7 @@ result = gencsupp.emit_void(result) return result - def OP_GETFIELD(self, op, ampersand=''): + def OP_GETFIELD(self, op, ampersand='', accessing_mem=True): assert isinstance(op.args[1], Constant) STRUCT = self.lltypemap(op.args[0]).TO structdef = self.db.gettypedefnode(STRUCT) @@ -472,7 +479,7 @@ expr = ampersand + structdef.ptr_access_expr(self.expr(op.args[0]), op.args[1].value, baseexpr_is_const) - return self.generic_get(op, expr) + return self.generic_get(op, expr, accessing_mem=accessing_mem) def OP_BARE_SETFIELD(self, op): assert isinstance(op.args[1], Constant) @@ -488,9 +495,9 @@ RESULT = self.lltypemap(op.result).TO if (isinstance(RESULT, FixedSizeArray) or (isinstance(RESULT, Array) and barebonearray(RESULT))): - return self.OP_GETFIELD(op, ampersand='') + return self.OP_GETFIELD(op, ampersand='', accessing_mem=False) else: - return self.OP_GETFIELD(op, ampersand='&') + return self.OP_GETFIELD(op, ampersand='&', accessing_mem=False) def OP_GETARRAYSIZE(self, op): ARRAY = self.lltypemap(op.args[0]).TO @@ -498,8 +505,7 @@ return '%s = %d;' % (self.expr(op.result), ARRAY.length) else: - return '%s = %s->length;' % (self.expr(op.result), - self.expr(op.args[0])) + return self.generic_get(op, '%s->length;' % self.expr(op.args[0])) def OP_GETARRAYITEM(self, op): ARRAY = self.lltypemap(op.args[0]).TO @@ -563,8 +569,7 @@ return '%s = %d;'%(self.expr(op.result), ARRAY.length) else: assert isinstance(ARRAY, Array) - return '%s = %s.length;'%(self.expr(op.result), expr) - + return self.generic_get(op, '%s.length;' % expr) def OP_PTR_NONZERO(self, op): @@ -608,18 +613,14 @@ return 'GC_REGISTER_FINALIZER(%s, (GC_finalization_proc)%s, NULL, NULL, NULL);' \ % (self.expr(op.args[0]), self.expr(op.args[1])) - def OP_RAW_MALLOC(self, op): - eresult = self.expr(op.result) - esize = self.expr(op.args[0]) - return "OP_RAW_MALLOC(%s, %s, void *);" % (esize, eresult) - def OP_STACK_MALLOC(self, op): - eresult = self.expr(op.result) - esize = self.expr(op.args[0]) - return "OP_STACK_MALLOC(%s, %s, void *);" % (esize, eresult) + #eresult = self.expr(op.result) + #esize = self.expr(op.args[0]) + #return "OP_STACK_MALLOC(%s, %s, void *);" % (esize, eresult) + raise NotImplementedError def OP_DIRECT_FIELDPTR(self, op): - return self.OP_GETFIELD(op, ampersand='&') + return self.OP_GETFIELD(op, ampersand='&', accessing_mem=False) def OP_DIRECT_ARRAYITEMS(self, op): ARRAY = self.lltypemap(op.args[0]).TO @@ -672,8 +673,10 @@ def OP_CAST_PTR_TO_INT(self, op): if self.db.reverse_debugger: - from rpython.translator.revdb import gencsupp - return gencsupp.cast_ptr_to_int(self, op) + TSRC = self.lltypemap(op.args[0]) + if isinstance(TSRC, Ptr) and TSRC.TO._gckind == 'gc': + from rpython.translator.revdb import gencsupp + return gencsupp.cast_gcptr_to_int(self, op) return self.OP_CAST_POINTER(op) def OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE(self, op): @@ -723,6 +726,7 @@ '((%(typename)s) (((char *)%(addr)s) + %(offset)s))[0] = %(value)s;' % locals()) OP_BARE_RAW_STORE = OP_RAW_STORE + OP_GC_STORE = OP_RAW_STORE # the difference is only in 'revdb_protect' def OP_RAW_LOAD(self, op): addr = self.expr(op.args[0]) diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -8,8 +8,8 @@ #define OP_STACK_CURRENT(r) r = (Signed)&r -#define OP_RAW_MALLOC(size, r, restype) { \ - r = (restype) malloc(size); \ +#define OP_RAW_MALLOC(size, r) { \ + r = malloc(size); \ if (r != NULL) { \ COUNT_MALLOC; \ } \ diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py --- a/rpython/translator/revdb/gencsupp.py +++ b/rpython/translator/revdb/gencsupp.py @@ -1,5 +1,6 @@ import py from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS from rpython.translator.c.support import cdecl from rpython.rlib import exports, revdb @@ -25,10 +26,13 @@ return 'rpy_reverse_db_register_destructor(%s, %s);' % ( funcgen.expr(op.args[0]), funcgen.expr(op.args[1])) -def cast_ptr_to_int(funcgen, op): +def cast_gcptr_to_int(funcgen, op): return '%s = RPY_REVDB_CAST_PTR_TO_INT(%s);' % ( funcgen.expr(op.result), funcgen.expr(op.args[0])) +set_revdb_protected = set(opname for opname, opdesc in LL_OPERATIONS.items() + if opdesc.revdb_protect) + def prepare_database(db): FUNCPTR = lltype.Ptr(lltype.FuncType([revdb._CMDPTR, lltype.Ptr(rstr.STR)], diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -966,7 +966,7 @@ memcpy(future_ids, extra, cmd->extra_size); future_ids[cmd->extra_size / sizeof(uint64_t)] = 0; uid_break = *future_ids; - attach_gdb(); + //attach_gdb(); } future_next_id = future_ids; } @@ -1274,8 +1274,6 @@ static void replay_call_destructors(void) { - fq_trigger(); - /* Re-enable fetching (disabled when we saw ASYNC_FINALIZER_TRIGGER), and fetch the uid's of dying objects with old-style destructors. */ @@ -1295,6 +1293,18 @@ obj = _ftree_pop(&destructor_tree, uid, &callback); callback(obj); } + + /* Now we're back in normal mode. We trigger the finalizer + queues here. */ + fq_trigger(); } /* ------------------------------------------------------------ */ + + +RPY_EXTERN +void seeing_uid(uint64_t uid) +{ + if (uid == 1895569) + attach_gdb(); +} diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h --- a/rpython/translator/revdb/src-revdb/revdb_include.h +++ b/rpython/translator/revdb/src-revdb/revdb_include.h @@ -37,8 +37,9 @@ ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)) #endif -#if 1 /* enable to print all allocs to stderr */ +#if 0 /* enable to print all allocs to stderr */ # define _RPY_REVDB_PRUID() \ + seeing_uid(uid); \ fprintf(stderr, \ "%s:%d: obj %llu\n", \ __FILE__, __LINE__, (unsigned long long) uid) diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py --- a/rpython/translator/revdb/test/test_basic.py +++ b/rpython/translator/revdb/test/test_basic.py @@ -83,6 +83,11 @@ def done(self): return self.cur == len(self.buffer) + def write_call(self, expected_string): + x = self.next() # raw_malloc: the pointer we got + x = self.next(); assert x == len(expected_string) + x = self.next('i'); assert x == 0 # errno + def compile(self, entry_point, backendopt=True, withsmallfuncsets=None): @@ -131,9 +136,7 @@ self.compile(main, backendopt=False) assert self.run('abc d') == '[abc, d]\n' rdb = self.fetch_rdb([self.exename, 'abc', 'd']) - # write() call - x = rdb.next(); assert x == len('[abc, d]\n') - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call('[abc, d]\n') x = rdb.next('q'); assert x == 0 # number of stop points # that's all we should get from this simple example assert rdb.done() @@ -151,9 +154,7 @@ hash_value = int(match.group(1)) rdb = self.fetch_rdb([self.exename, 'Xx']) # compute_identity_hash() doesn't record anything - # write() call - x = rdb.next(); assert x == len(out) - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call(out) # done x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() @@ -174,8 +175,7 @@ # write() call (it used to be the case that vtable reads where # recorded too; the single byte fetched from the vtable from # the '.x' in main() would appear here) - x = rdb.next(); assert x == len(out) - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call(out) # done x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() @@ -195,8 +195,7 @@ assert out == '41\n' rdb = self.fetch_rdb([self.exename, 'Xx']) # write() call - x = rdb.next(); assert x == len(out) - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call(out) # done x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() @@ -230,8 +229,7 @@ assert out == expected_output rdb = self.fetch_rdb([self.exename] + input.split()) # write() call - x = rdb.next(); assert x == len(out) - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call(out) x = rdb.next('q'); assert x == 0 # number of stop points assert rdb.done() diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py --- a/rpython/translator/revdb/test/test_weak.py +++ b/rpython/translator/revdb/test/test_weak.py @@ -251,9 +251,7 @@ x = rdb.next() assert x == len(seen_uids) assert len(seen_uids) == int(out) - # write() call - x = rdb.next(); assert x == len(out) - x = rdb.next('i'); assert x == 0 # errno + rdb.write_call(out) x = rdb.next('q'); assert x == 3000 # number of stop points From pypy.commits at gmail.com Thu Jun 30 08:31:11 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 30 Jun 2016 05:31:11 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger: Remove debugging output Message-ID: <5775110f.871a1c0a.eb5f5.fffffdb0@mx.google.com> Author: Armin Rigo Branch: reverse-debugger Changeset: r85470:c4e9f3b2065e Date: 2016-06-30 14:32 +0200 http://bitbucket.org/pypy/pypy/changeset/c4e9f3b2065e/ Log: Remove debugging output diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -1229,8 +1229,8 @@ RPY_EXTERN int rpy_reverse_db_fq_register(void *obj) { - fprintf(stderr, "FINALIZER_TREE: %lld -> %p\n", - ((struct pypy_header0 *)obj)->h_uid, obj); + /*fprintf(stderr, "FINALIZER_TREE: %lld -> %p\n", + ((struct pypy_header0 *)obj)->h_uid, obj);*/ if (!RPY_RDB_REPLAY) { return 0; /* recording */ } @@ -1247,7 +1247,7 @@ int64_t uid; RPY_REVDB_EMIT(uid = result ? ((struct pypy_header0 *)result)->h_uid : -1;, int64_t _e, uid); - fprintf(stderr, "next_dead: object %lld\n", uid); + /*fprintf(stderr, "next_dead: object %lld\n", uid);*/ if (RPY_RDB_REPLAY) { if (uid == -1) { result = NULL; From pypy.commits at gmail.com Thu Jun 30 08:42:32 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 30 Jun 2016 05:42:32 -0700 (PDT) Subject: [pypy-commit] pypy default: don't add superfluous QUASIIMMUT_FIELD during tracing (I've seen their creation Message-ID: <577513b8.949a1c0a.342ea.ffffd1cd@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r85471:b116f09c4e9d Date: 2016-06-30 14:41 +0200 http://bitbucket.org/pypy/pypy/changeset/b116f09c4e9d/ Log: don't add superfluous QUASIIMMUT_FIELD during tracing (I've seen their creation take 2% of the full execution time during profiling) diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -57,10 +57,16 @@ self.cache_anything = {} self.cache_seen_allocation = {} + # set of boxes that we've seen a quasi-immut for the field on. cleared + # on writes to the field. + self.quasiimmut_seen = None + def _clear_cache_on_write(self, seen_allocation_of_target): if not seen_allocation_of_target: self.cache_seen_allocation.clear() self.cache_anything.clear() + if self.quasiimmut_seen is not None: + self.quasiimmut_seen.clear() def _seen_alloc(self, ref_box): if not isinstance(ref_box, RefFrontendOp): @@ -92,6 +98,8 @@ def invalidate_unescaped(self): self._invalidate_unescaped(self.cache_anything) self._invalidate_unescaped(self.cache_seen_allocation) + if self.quasiimmut_seen is not None: + self.quasiimmut_seen.clear() def _invalidate_unescaped(self, d): for ref_box in d.keys(): @@ -484,3 +492,18 @@ if isinstance(oldbox, FrontendOp) and isinstance(newbox, Const): assert newbox.same_constant(constant_from_op(oldbox)) oldbox.set_replaced_with_const() + + def is_quasi_immut_known(self, fielddescr, box): + cache = self.heap_cache.get(fielddescr, None) + if cache is not None and cache.quasiimmut_seen is not None: + return box in cache.quasiimmut_seen + return False + + def quasi_immut_now_known(self, fielddescr, box): + cache = self.heap_cache.get(fielddescr, None) + if cache is None: + cache = self.heap_cache[fielddescr] = CacheEntry(self) + if cache.quasiimmut_seen is not None: + cache.quasiimmut_seen[box] = None + else: + cache.quasiimmut_seen = {box: None} diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -829,8 +829,11 @@ mutatefielddescr, orgpc): from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr cpu = self.metainterp.cpu + if self.metainterp.heapcache.is_quasi_immut_known(fielddescr, box): + return descr = QuasiImmutDescr(cpu, box.getref_base(), fielddescr, mutatefielddescr) + self.metainterp.heapcache.quasi_immut_now_known(fielddescr, box) self.metainterp.history.record(rop.QUASIIMMUT_FIELD, [box], None, descr=descr) self.metainterp.generate_guard(rop.GUARD_NOT_INVALIDATED, diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py --- a/rpython/jit/metainterp/test/test_heapcache.py +++ b/rpython/jit/metainterp/test/test_heapcache.py @@ -797,3 +797,46 @@ h.class_now_known(box1) # interaction of the two families of flags assert not h.is_unescaped(box1) assert h.is_likely_virtual(box1) + + def test_quasiimmut_seen(self): + h = HeapCache() + box1 = RefFrontendOp(1) + box2 = RefFrontendOp(2) + box3 = RefFrontendOp(3) + box4 = RefFrontendOp(4) + assert not h.is_quasi_immut_known(descr1, box1) + assert not h.is_quasi_immut_known(descr1, box2) + assert not h.is_quasi_immut_known(descr2, box3) + assert not h.is_quasi_immut_known(descr2, box4) + h.quasi_immut_now_known(descr1, box1) + assert h.is_quasi_immut_known(descr1, box1) + assert not h.is_quasi_immut_known(descr1, box2) + assert not h.is_quasi_immut_known(descr2, box3) + assert not h.is_quasi_immut_known(descr2, box4) + h.quasi_immut_now_known(descr1, box2) + assert h.is_quasi_immut_known(descr1, box1) + assert h.is_quasi_immut_known(descr1, box2) + assert not h.is_quasi_immut_known(descr2, box3) + assert not h.is_quasi_immut_known(descr2, box4) + h.quasi_immut_now_known(descr2, box3) + assert h.is_quasi_immut_known(descr1, box1) + assert h.is_quasi_immut_known(descr1, box2) + assert h.is_quasi_immut_known(descr2, box3) + assert not h.is_quasi_immut_known(descr2, box4) + h.quasi_immut_now_known(descr2, box4) + assert h.is_quasi_immut_known(descr1, box1) + assert h.is_quasi_immut_known(descr1, box2) + assert h.is_quasi_immut_known(descr2, box3) + assert h.is_quasi_immut_known(descr2, box4) + + # invalidate the descr1 cache + + h.setfield(box1, box3, descr1) + assert not h.is_quasi_immut_known(descr1, box1) + assert not h.is_quasi_immut_known(descr1, box2) + + # a call invalidates everything + h.invalidate_caches( + rop.CALL_N, FakeCallDescr(FakeEffectinfo.EF_CAN_RAISE), []) + assert not h.is_quasi_immut_known(descr2, box3) + assert not h.is_quasi_immut_known(descr2, box4) diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py --- a/rpython/jit/metainterp/test/test_tracingopts.py +++ b/rpython/jit/metainterp/test/test_tracingopts.py @@ -512,6 +512,32 @@ assert res == 4 * 7 self.check_operations_history(getfield_gc_i=2, getfield_gc_r=2) + def test_heap_caching_quasi_immutable(self): + class A: + _immutable_fields_ = ['x?'] + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 7 + + @jit.elidable + def get(n): + if n > 0: + return a1 + return a2 + + def g(a): + return a.x + + def fn(n): + jit.promote(n) + a = get(n) + return g(a) + a.x + res = self.interp_operations(fn, [7]) + assert res == 10 + self.check_operations_history(quasiimmut_field=1) + + def test_heap_caching_multiple_tuples(self): class Gbl(object): pass From pypy.commits at gmail.com Thu Jun 30 08:47:38 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 30 Jun 2016 05:47:38 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: print the implicit conditions at the beginning of a bridge too Message-ID: <577514ea.c5461c0a.cee39.7aac@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85472:4c174b7fa972 Date: 2016-06-29 14:52 +0200 http://bitbucket.org/pypy/pypy/changeset/4c174b7fa972/ Log: print the implicit conditions at the beginning of a bridge too diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -182,6 +182,13 @@ return "\n".join([cond.repr(argrepr) for cond in self.conditions]) + def repr_of_conditions_as_jit_debug(self, argrepr="?"): + conditions = [cond.repr(argrepr) for cond in self.conditions] + # make fake jit-debug ops to print + for i in range(len(conditions)): + conditions[i] = "jit_debug('%s')" % (conditions[i], ) + return conditions + class Condition(object): def __init__(self, optimizer): diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1163,6 +1163,12 @@ return self._compatibility_conditions.repr_of_conditions(argrepr) return '' + def repr_of_conditions_as_jit_debug(self, argrepr="?"): + if self._compatibility_conditions: + return self._compatibility_conditions.repr_of_conditions_as_jit_debug(argrepr) + return [] + + # ____________________________________________________________ memory_error = MemoryError() diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py --- a/rpython/jit/metainterp/logger.py +++ b/rpython/jit/metainterp/logger.py @@ -57,6 +57,7 @@ def log_bridge(self, inputargs, operations, extra=None, descr=None, ops_offset=None, memo=None): + from rpython.jit.metainterp.compile import GuardCompatibleDescr if extra == "noopt": debug_start("jit-log-noopt-bridge") debug_print("# bridge out of Guard", @@ -80,11 +81,23 @@ debug_stop("jit-log-compiling-bridge") else: debug_start("jit-log-opt-bridge") - debug_print("# bridge out of Guard", - "0x%x" % r_uint(compute_unique_id(descr)), - "with", len(operations), "ops") - logops = self._log_operations(inputargs, operations, ops_offset, - memo) + if have_debug_prints(): + print_after_inputargs = '' + debug_print("# bridge out of Guard", + "0x%x" % r_uint(compute_unique_id(descr)), + "with", len(operations), "ops") + logops = self._make_log_operations(memo) + if isinstance(descr, GuardCompatibleDescr): + if descr.fallback_jump_target == 0: + # this means it's the last attached guard + ccond = descr.other_compat_conditions[-1] + argrepr = logops.repr_of_arg( + inputargs[descr.failarg_index]) + conditions = ccond.repr_of_conditions_as_jit_debug( + argrepr) + print_after_inputargs = "\n".join(conditions) + logops = self._log_operations(inputargs, operations, ops_offset, + memo, logops, print_after_inputargs) debug_stop("jit-log-opt-bridge") return logops @@ -105,11 +118,12 @@ debug_stop("jit-abort-log") return logops - def _log_operations(self, inputargs, operations, ops_offset, memo=None): + def _log_operations(self, inputargs, operations, ops_offset, memo=None, logops=None, print_after_inputargs=''): if not have_debug_prints(): return None - logops = self._make_log_operations(memo) - logops._log_operations(inputargs, operations, ops_offset, memo) + if logops is None: + logops = self._make_log_operations(memo) + logops._log_operations(inputargs, operations, ops_offset, memo, print_after_inputargs) return logops def _make_log_operations(self, memo): @@ -229,18 +243,14 @@ from rpython.jit.metainterp.compile import GuardCompatibleDescr descr = op.getdescr() assert isinstance(descr, GuardCompatibleDescr) - conditions = descr.repr_of_conditions(argreprs[0]) + conditions = descr.repr_of_conditions_as_jit_debug(argreprs[0]) if conditions: - # make fake jit-debug ops to print - conditions = conditions.split("\n") - for i in range(len(conditions)): - conditions[i] = "jit_debug('%s')" % (conditions[i], ) fail_args += "\n" + "\n".join(conditions) return s_offset + res + op.getopname() + '(' + args + ')' + fail_args def _log_operations(self, inputargs, operations, ops_offset=None, - memo=None): + memo=None, print_after_inputargs=''): if not have_debug_prints(): return if ops_offset is None: @@ -248,6 +258,8 @@ if inputargs is not None: args = ", ".join([self.repr_of_arg(arg) for arg in inputargs]) debug_print('[' + args + ']') + if print_after_inputargs: + debug_print(print_after_inputargs) for i in range(len(operations)): #op = operations[i] debug_print(self.repr_of_resop(operations[i], ops_offset)) From pypy.commits at gmail.com Thu Jun 30 08:47:42 2016 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 30 Jun 2016 05:47:42 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: merge default Message-ID: <577514ee.d11b1c0a.aa442.068d@mx.google.com> Author: Carl Friedrich Bolz Branch: guard-compatible Changeset: r85473:900ff3842617 Date: 2016-06-30 14:46 +0200 http://bitbucket.org/pypy/pypy/changeset/900ff3842617/ Log: merge default diff too long, truncating to 2000 out of 2999 lines diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1036,7 +1036,7 @@ return (None, None) def newlist_bytes(self, list_s): - return self.newlist([self.wrap(s) for s in list_s]) + return self.newlist([self.newbytes(s) for s in list_s]) def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) @@ -1535,7 +1535,7 @@ # unclear if there is any use at all for getting the bytes in # the unicode buffer.) try: - return self.str_w(w_obj) + return self.bytes_w(w_obj) except OperationError as e: if not e.match(self, self.w_TypeError): raise diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -408,14 +408,14 @@ w(self.co_nlocals), w(self.co_stacksize), w(self.co_flags), - w(self.co_code), + space.newbytes(self.co_code), space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), w(self.co_name), w(self.co_firstlineno), - w(self.co_lnotab), + space.newbytes(self.co_lnotab), space.newtuple([w(v) for v in self.co_freevars]), space.newtuple([w(v) for v in self.co_cellvars]), w(self.magic), diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -13,12 +13,11 @@ # PYPY Modification: removed the EMPTY class as it's not needed here -# PYPY Modification: we don't need a particuliar DEFAULT class here -# a simple None works fine. -# (Having a DefaultClass inheriting from str makes -# the annotator crash) -DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys - # anyway using dicts doesn't seem the best final way to store these char indexed tables +# PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython +# dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into +# more compact strings. +DEFAULT = object() + # PYPY Modification : removed all automata functions (any, maybe, # newArcPair, etc.) diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -293,7 +293,7 @@ i = 0 for k, v in sorted(state.items()): i += 1 - if k == '\x00default': + if k == DEFAULT: k = "automata.DEFAULT" else: k = repr(k) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -81,7 +81,7 @@ if need_encoding: enc = encoding v = PyString_DecodeEscape(space, substr, 'strict', enc) - return space.wrap(v) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -84,7 +84,7 @@ if self.size == 1: with cdataobj as ptr: s = ptr[0] - return self.space.wrap(s) + return self.space.newbytes(s) return W_CType.string(self, cdataobj, maxlen) def unpack_ptr(self, w_ctypeptr, ptr, length): @@ -126,12 +126,12 @@ return self.space.wrap(ord(cdata[0])) def convert_to_object(self, cdata): - return self.space.wrap(cdata[0]) + return self.space.newbytes(cdata[0]) def _convert_to_char(self, w_ob): space = self.space if space.isinstance_w(w_ob, space.w_str): - s = space.str_w(w_ob) + s = space.bytes_w(w_ob) if len(s) == 1: return s[0] if (isinstance(w_ob, cdataobj.W_CData) and @@ -146,7 +146,7 @@ def unpack_ptr(self, w_ctypeptr, ptr, length): s = rffi.charpsize2str(ptr, length) - return self.space.wrapbytes(s) + return self.space.newbytes(s) # XXX explicitly use an integer type instead of lltype.UniChar here, diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -120,7 +120,7 @@ s = rffi.charp2str(ptr) else: s = rffi.charp2strn(ptr, length) - return space.wrapbytes(s) + return space.newbytes(s) # # pointer to a wchar_t: builds and returns a unicode if self.is_unichar_ptr_or_array(): @@ -129,7 +129,7 @@ u = rffi.wcharp2unicode(cdata) else: u = rffi.wcharp2unicoden(cdata, length) - return space.wrap(u) + return space.newunicode(u) # return W_CType.string(self, cdataobj, maxlen) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -36,12 +36,14 @@ w_errorhandler = lookup_error(space, errors) if decode: w_cls = space.w_UnicodeDecodeError + w_input = space.newbytes(input) else: w_cls = space.w_UnicodeEncodeError + w_input = space.newunicode(input) w_exc = space.call_function( w_cls, space.wrap(encoding), - space.wrap(input), + w_input, space.wrap(startpos), space.wrap(endpos), space.wrap(reason)) @@ -314,7 +316,7 @@ @unwrap_spec(errors='str_or_None') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) - return space.newtuple([space.wrap(s), space.wrap(len(s))]) + return space.newtuple([space.newbytes(s), space.wrap(len(s))]) @unwrap_spec(errors='str_or_None') def charbuffer_encode(space, w_data, errors='strict'): @@ -377,7 +379,7 @@ state = space.fromcache(CodecState) func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) wrap_encoder.func_name = rname globals()[name] = wrap_encoder @@ -398,7 +400,7 @@ wrap_decoder.func_name = rname globals()[name] = wrap_decoder -for encoders in [ +for encoder in [ "ascii_encode", "latin_1_encode", "utf_7_encode", @@ -412,9 +414,9 @@ "raw_unicode_escape_encode", "unicode_internal_encode", ]: - make_encoder_wrapper(encoders) + make_encoder_wrapper(encoder) -for decoders in [ +for decoder in [ "ascii_decode", "latin_1_decode", "utf_7_decode", @@ -426,7 +428,7 @@ "utf_32_le_decode", "raw_unicode_escape_decode", ]: - make_decoder_wrapper(decoders) + make_decoder_wrapper(decoder) if hasattr(runicode, 'str_decode_mbcs'): make_encoder_wrapper('mbcs_encode') @@ -560,7 +562,7 @@ if space.isinstance_w(w_ch, space.w_str): # Charmap may return a string - return space.str_w(w_ch) + return space.bytes_w(w_ch) elif space.isinstance_w(w_ch, space.w_int): # Charmap may return a number x = space.int_w(w_ch) @@ -608,7 +610,7 @@ result = runicode.unicode_encode_charmap( uni, len(uni), errors, state.encode_error_handler, mapping) - return space.newtuple([space.wrap(result), space.wrap(len(uni))]) + return space.newtuple([space.newbytes(result), space.wrap(len(uni))]) @unwrap_spec(chars=unicode) @@ -696,4 +698,4 @@ def escape_decode(space, data, errors='strict'): from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, errors, None) - return space.newtuple([space.wrap(result), space.wrap(len(data))]) + return space.newtuple([space.newbytes(result), space.wrap(len(data))]) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -111,7 +111,7 @@ def digest(self, space): "Return the digest value as a string of binary data." digest = self._digest(space) - return space.wrap(digest) + return space.newbytes(digest) def hexdigest(self, space): "Return the digest value as a string of hexadecimal digits." diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -343,7 +343,7 @@ self._writer_reset_buf() def _write(self, space, data): - w_data = space.wrap(data) + w_data = space.newbytes(data) while True: try: w_written = space.call_method(self.w_raw, "write", w_data) @@ -415,7 +415,7 @@ else: raise oefmt(space.w_ValueError, "read length must be positive or -1") - return space.wrap(res) + return space.newbytes(res) @unwrap_spec(size=int) def peek_w(self, space, size=0): @@ -432,7 +432,7 @@ have = self._readahead() if have > 0: data = ''.join(self.buffer[self.pos:self.pos+have]) - return space.wrap(data) + return space.newbytes(data) # Fill the buffer from the raw stream, and copy it to the result self._reader_reset_buf() @@ -442,7 +442,7 @@ size = 0 self.pos = 0 data = ''.join(self.buffer[:size]) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(size=int) def read1_w(self, space, size): @@ -452,7 +452,7 @@ if size < 0: raise oefmt(space.w_ValueError, "read length must be positive") if size == 0: - return space.wrap("") + return space.newbytes("") with self.lock: # Return up to n bytes. If at least one byte is buffered, we only @@ -480,7 +480,7 @@ endpos = self.pos + size data = ''.join(self.buffer[self.pos:endpos]) self.pos = endpos - return space.wrap(data) + return space.newbytes(data) def _read_all(self, space): "Read all the file, don't update the cache" @@ -505,7 +505,7 @@ if current_size == 0: return w_data break - data = space.str_w(w_data) + data = space.bytes_w(w_data) size = len(data) if size == 0: break @@ -513,7 +513,7 @@ current_size += size if self.abs_pos != -1: self.abs_pos += size - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def _raw_read(self, space, buffer, start, length): length = intmask(length) @@ -644,11 +644,11 @@ else: pos = -1 if pos >= 0: - w_res = space.wrap(''.join(self.buffer[self.pos:pos+1])) + w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1])) self.pos = pos + 1 return w_res if have == limit: - w_res = space.wrap(''.join(self.buffer[self.pos:self.pos+have])) + w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have])) self.pos += have return w_res @@ -690,7 +690,7 @@ written += have if limit >= 0: limit -= have - return space.wrap(''.join(chunks)) + return space.newbytes(''.join(chunks)) # ____________________________________________________ # Write methods @@ -1024,7 +1024,6 @@ self._deprecated_max_buffer_size(space) self.state = STATE_ZERO - check_readable_w(space, w_raw) check_writable_w(space, w_raw) check_seekable_w(space, w_raw) diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -32,12 +32,12 @@ def read_w(self, space, w_size=None): self._check_closed(space) size = convert_size(space, w_size) - return space.wrap(self.read(size)) + return space.newbytes(self.read(size)) def readline_w(self, space, w_limit=None): self._check_closed(space) limit = convert_size(space, w_limit) - return space.wrap(self.readline(limit)) + return space.newbytes(self.readline(limit)) def read1_w(self, space, w_size): return self.read_w(space, w_size) @@ -81,7 +81,7 @@ def getvalue_w(self, space): self._check_closed(space) - return space.wrap(self.getvalue()) + return space.newbytes(self.getvalue()) def tell_w(self, space): self._check_closed(space) @@ -128,7 +128,7 @@ def getstate_w(self, space): self._check_closed(space) return space.newtuple([ - space.wrap(self.getvalue()), + space.newbytes(self.getvalue()), space.wrap(self.tell()), self.getdict(space)]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -361,7 +361,7 @@ raise wrap_oserror(space, e, exception_name='w_IOError') - return space.wrap(s) + return space.newbytes(s) def readinto_w(self, space, w_buffer): self._check_closed(space) @@ -405,7 +405,7 @@ break builder.append(chunk) total += len(chunk) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) if sys.platform == "win32": def _truncate(self, size): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -192,7 +192,7 @@ length = space.len_w(w_readahead) if length > 0: n = 0 - buf = space.str_w(w_readahead) + buf = space.bytes_w(w_readahead) if limit >= 0: while True: if n >= length or n >= limit: @@ -219,7 +219,7 @@ raise oefmt(space.w_IOError, "peek() should have returned a bytes object, not " "'%T'", w_read) - read = space.str_w(w_read) + read = space.bytes_w(w_read) if not read: break @@ -229,7 +229,7 @@ if read[-1] == '\n': break - return space.wrap(builder.build()) + return space.newbytes(builder.build()) def readlines_w(self, space, w_hint=None): hint = convert_size(space, w_hint) @@ -339,11 +339,11 @@ if not space.isinstance_w(w_data, space.w_str): raise oefmt(space.w_TypeError, "read() should return bytes") - data = space.str_w(w_data) + data = space.bytes_w(w_data) if not data: break builder.append(data) - return space.wrap(builder.build()) + return space.newbytes(builder.build()) W_RawIOBase.typedef = TypeDef( '_io._RawIOBase', W_IOBase.typedef, diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -160,7 +160,7 @@ w_buffer, w_flag = space.unpackiterable(w_state, 2) flag = space.r_longlong_w(w_flag) else: - w_buffer = space.wrap("") + w_buffer = space.newbytes("") flag = 0 flag <<= 1 if self.pendingcr: @@ -556,7 +556,7 @@ # Given this, we know there was a valid snapshot point # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). w_dec_buffer, w_dec_flags = space.unpackiterable(w_state, 2) - dec_buffer = space.str_w(w_dec_buffer) + dec_buffer = space.bytes_w(w_dec_buffer) dec_flags = space.int_w(w_dec_flags) else: dec_buffer = None @@ -582,7 +582,7 @@ if self.telling: # At the snapshot point, len(dec_buffer) bytes before the read, # the next input to be decoded is dec_buffer + input_chunk. - next_input = dec_buffer + space.str_w(w_input) + next_input = dec_buffer + space.bytes_w(w_input) self.snapshot = PositionSnapshot(dec_flags, next_input) return not eof @@ -769,7 +769,7 @@ else: w_bytes = space.call_method(self.w_encoder, "encode", w_text) - b = space.str_w(w_bytes) + b = space.bytes_w(w_bytes) if not self.pending_bytes: self.pending_bytes = [] self.pending_bytes_count = 0 @@ -799,7 +799,8 @@ while True: try: - space.call_method(self.w_buffer, "write", space.wrap(pending_bytes)) + space.call_method(self.w_buffer, "write", + space.newbytes(pending_bytes)) except OperationError as e: if trap_eintr(space, e): continue @@ -828,7 +829,7 @@ space.call_method(self.w_decoder, "reset") else: space.call_method(self.w_decoder, "setstate", - space.newtuple([space.wrap(""), + space.newtuple([space.newbytes(""), space.wrap(cookie.dec_flags)])) def _encoder_setstate(self, space, cookie): @@ -904,7 +905,7 @@ raise oefmt(space.w_TypeError, msg, w_chunk) self.snapshot = PositionSnapshot(cookie.dec_flags, - space.str_w(w_chunk)) + space.bytes_w(w_chunk)) w_decoded = space.call_method(self.w_decoder, "decode", w_chunk, space.wrap(cookie.need_eof)) @@ -975,7 +976,7 @@ i = 0 while i < len(input): w_decoded = space.call_method(self.w_decoder, "decode", - space.wrap(input[i])) + space.newbytes(input[i])) check_decoded(space, w_decoded) chars_decoded += len(space.unicode_w(w_decoded)) diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -20,7 +20,7 @@ self.update(string) def digest_w(self): - return self.space.wrap(self.digest()) + return self.space.newbytes(self.digest()) def hexdigest_w(self): return self.space.wrap(self.hexdigest()) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -83,12 +83,12 @@ return space.w_None except curses_error as e: raise convert_error(space, e) - return space.wrap(result) + return space.newbytes(result) @unwrap_spec(s=str) def tparm(space, s, args_w): args = [space.int_w(a) for a in args_w] try: - return space.wrap(_curses_tparm(s, args)) + return space.newbytes(_curses_tparm(s, args)) except curses_error as e: raise convert_error(space, e) diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -113,7 +113,7 @@ pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) assert 0 <= pos <= len(object) self.pending = object[pos:] - return space.wrap(output) + return space.newbytes(output) @unwrap_spec(errors="str_or_None") diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -40,7 +40,7 @@ raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.wrap(output), + return space.newtuple([space.newbytes(output), space.wrap(len(input))]) @@ -66,7 +66,7 @@ space.w_UnicodeDecodeError, space.newtuple([ space.wrap(name), - space.wrap(input), + space.newbytes(input), space.wrap(e.start), space.wrap(e.end), space.wrap(e.reason)])) diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -122,9 +122,9 @@ space, self.BUFFER_SIZE, maxlength) try: if newbuf: - return space.wrap(rffi.charpsize2str(newbuf, res)) + return space.newbytes(rffi.charpsize2str(newbuf, res)) else: - return space.wrap(rffi.charpsize2str(self.buffer, res)) + return space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) @@ -138,7 +138,7 @@ space, length - offset, PY_SSIZE_T_MAX) try: if newbuf: - raise BufferTooShort(space, space.wrap( + raise BufferTooShort(space, space.newbytes( rffi.charpsize2str(newbuf, res))) rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res)) finally: @@ -166,9 +166,9 @@ space, self.BUFFER_SIZE, PY_SSIZE_T_MAX) try: if newbuf: - w_received = space.wrap(rffi.charpsize2str(newbuf, res)) + w_received = space.newbytes(rffi.charpsize2str(newbuf, res)) else: - w_received = space.wrap(rffi.charpsize2str(self.buffer, res)) + w_received = space.newbytes(rffi.charpsize2str(self.buffer, res)) finally: if newbuf: rffi.free_charp(newbuf) diff --git a/pypy/module/_rawffi/alt/test/test_type_converter.py b/pypy/module/_rawffi/alt/test/test_type_converter.py --- a/pypy/module/_rawffi/alt/test/test_type_converter.py +++ b/pypy/module/_rawffi/alt/test/test_type_converter.py @@ -12,14 +12,14 @@ handle_signed = handle_all handle_unsigned = handle_all handle_pointer = handle_all - handle_char = handle_all + handle_char = handle_all handle_unichar = handle_all handle_longlong = handle_all handle_char_p = handle_all handle_unichar_p = handle_all handle_float = handle_all handle_singlefloat = handle_all - + def handle_struct(self, w_ffitype, w_structinstance): self.lastval = w_structinstance @@ -119,12 +119,12 @@ def test_strings(self): # first, try automatic conversion from applevel - self.check(app_types.char_p, self.space.wrap('foo'), 'foo') - self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') - self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') + self.check(app_types.char_p, self.space.newbytes('foo'), 'foo') + self.check(app_types.unichar_p, self.space.wrap(u'foo\u1234'), u'foo\u1234') + self.check(app_types.unichar_p, self.space.wrap('foo'), u'foo') # then, try to pass explicit pointers self.check(app_types.char_p, self.space.wrap(42), 42) - self.check(app_types.unichar_p, self.space.wrap(42), 42) + self.check(app_types.unichar_p, self.space.wrap(42), 42) @@ -136,7 +136,7 @@ get_signed = get_all get_unsigned = get_all get_pointer = get_all - get_char = get_all + get_char = get_all get_unichar = get_all get_longlong = get_all get_char_p = get_all @@ -144,7 +144,7 @@ get_float = get_all get_singlefloat = get_all get_unsigned_which_fits_into_a_signed = get_all - + def convert(self, w_ffitype, val): self.val = val return self.do_and_wrap(w_ffitype) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -181,7 +181,7 @@ start, stop = self.decodeslice(space, w_slice) ll_buffer = self.ll_buffer result = [ll_buffer[i] for i in range(start, stop)] - return space.wrap(''.join(result)) + return space.newbytes(''.join(result)) def setslice(self, space, w_slice, w_value): start, stop = self.decodeslice(space, w_slice) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -570,7 +570,7 @@ s = rffi.charp2str(charp_addr) else: s = rffi.charp2strn(charp_addr, maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2unicode(space, address, maxlength=-1): @@ -588,7 +588,7 @@ if maxlength == -1: return charp2string(space, address) s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength) - return space.wrap(s) + return space.newbytes(s) @unwrap_spec(address=r_uint, maxlength=int) def wcharp2rawunicode(space, address, maxlength=-1): diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -209,7 +209,7 @@ buf = rsocket.inet_aton(ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(packed=str) def inet_ntoa(space, packed): @@ -234,7 +234,7 @@ buf = rsocket.inet_pton(family, ip) except SocketError as e: raise converted_error(space, e) - return space.wrap(buf) + return space.newbytes(buf) @unwrap_spec(family=int, packed=str) def inet_ntop(space, family, packed): @@ -263,10 +263,10 @@ if space.is_w(w_host, space.w_None): host = None elif space.isinstance_w(w_host, space.w_str): - host = space.str_w(w_host) + host = space.bytes_w(w_host) elif space.isinstance_w(w_host, space.w_unicode): w_shost = space.call_method(w_host, "encode", space.wrap("idna")) - host = space.str_w(w_shost) + host = space.bytes_w(w_shost) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 1 must be string or None") @@ -277,7 +277,7 @@ elif space.isinstance_w(w_port, space.w_int) or space.isinstance_w(w_port, space.w_long): port = str(space.int_w(w_port)) elif space.isinstance_w(w_port, space.w_str): - port = space.str_w(w_port) + port = space.bytes_w(w_port) else: raise oefmt(space.w_TypeError, "getaddrinfo() argument 2 must be integer or string") diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -296,7 +296,7 @@ except SocketError as e: raise converted_error(space, e) buflen = space.int_w(w_buflen) - return space.wrap(self.sock.getsockopt(level, optname, buflen)) + return space.newbytes(self.sock.getsockopt(level, optname, buflen)) def gettimeout_w(self, space): """gettimeout() -> timeout @@ -345,7 +345,7 @@ data = self.sock.recv(buffersize, flags) except SocketError as e: raise converted_error(space, e) - return space.wrap(data) + return space.newbytes(data) @unwrap_spec(buffersize='nonnegint', flags=int) def recvfrom_w(self, space, buffersize, flags=0): @@ -359,7 +359,7 @@ w_addr = addr_as_object(addr, self.sock.fd, space) else: w_addr = space.w_None - return space.newtuple([space.wrap(data), w_addr]) + return space.newtuple([space.newbytes(data), w_addr]) except SocketError as e: raise converted_error(space, e) @@ -436,7 +436,7 @@ except OperationError as e: if e.async(space): raise - optval = space.str_w(w_optval) + optval = space.bytes_w(w_optval) try: self.sock.setsockopt(level, optname, optval) except SocketError as e: diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -36,11 +36,12 @@ def slice_w(space, ctx, start, end, w_default): if 0 <= start <= end: if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.getslice(start, end, 1, end-start)) + return space.newbytes(ctx._buffer.getslice(start, end, 1, + end-start)) if isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string[start:end]) + return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr[start:end]) + return space.newunicode(ctx._unicodestr[start:end]) else: # unreachable raise SystemError @@ -242,7 +243,7 @@ space.isinstance_w(w_string, space.w_unicode) and literal) else: try: - filter_as_string = space.str_w(w_ptemplate) + filter_as_string = space.bytes_w(w_ptemplate) except OperationError as e: if e.async(space): raise @@ -331,15 +332,15 @@ strbuilder, unicodebuilder, last_pos, ctx.end) if use_builder: if strbuilder is not None: - return space.wrap(strbuilder.build()), n + return space.newbytes(strbuilder.build()), n else: assert unicodebuilder is not None - return space.wrap(unicodebuilder.build()), n + return space.newunicode(unicodebuilder.build()), n else: if space.isinstance_w(w_string, space.w_unicode): - w_emptystr = space.wrap(u'') + w_emptystr = space.newunicode(u'') else: - w_emptystr = space.wrap('') + w_emptystr = space.newbytes('') w_item = space.call_method(w_emptystr, 'join', space.newlist(sublist_w)) return w_item, n @@ -565,11 +566,11 @@ def fget_string(self, space): ctx = self.ctx if isinstance(ctx, rsre_core.BufMatchContext): - return space.wrap(ctx._buffer.as_str()) + return space.newbytes(ctx._buffer.as_str()) elif isinstance(ctx, rsre_core.StrMatchContext): - return space.wrap(ctx._string) + return space.newbytes(ctx._string) elif isinstance(ctx, rsre_core.UnicodeMatchContext): - return space.wrap(ctx._unicodestr) + return space.newunicode(ctx._unicodestr) else: raise SystemError diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -666,7 +666,7 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrap(rffi.charpsize2str(buf, intmask(length))) + return space.newbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx @@ -707,7 +707,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + return space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -926,7 +926,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - w_value = space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length)) w_value = space.call_method(w_value, "decode", space.wrap("utf-8")) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1232,7 +1232,7 @@ w_ssl_socket, space.w_None, w_ctx) else: - w_servername = space.wrapbytes(rffi.charp2str(servername)) + w_servername = space.newbytes(rffi.charp2str(servername)) try: w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) @@ -1778,7 +1778,7 @@ if not path: return space.w_None else: - return space.wrapbytes(rffi.charp2str(path)) + return space.newbytes(rffi.charp2str(path)) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,7 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, + if not CertGetEnhancedKeyUsage(pCertCtx, flags, lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: @@ -120,7 +120,7 @@ pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx) if not pCertCtx: break - w_cert = space.wrapbytes( + w_cert = space.newbytes( rffi.charpsize2str(pCertCtx.c_pbCertEncoded, intmask(pCertCtx.c_cbCertEncoded))) w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType) @@ -162,7 +162,7 @@ pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx) if not pCrlCtx: break - w_crl = space.wrapbytes( + w_crl = space.newbytes( rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded, intmask(pCrlCtx.c_cbCrlEncoded))) w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -380,7 +380,7 @@ return space.newlist(l) else: # REG_BINARY and all other types - return space.wrap(rffi.charpsize2str(buf, buflen)) + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name=str, typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -225,11 +225,11 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.newbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() - return self.space.wrap(s) + return self.space.newbytes(s) def descr_fromstring(self, space, w_s): """ fromstring(string) @@ -263,7 +263,7 @@ except OverflowError: raise MemoryError w_item = space.call_method(w_f, 'read', space.wrap(size)) - item = space.str_w(w_item) + item = space.bytes_w(w_item) if len(item) < size: n = len(item) % self.itemsize elems = max(0, len(item) - (len(item) % self.itemsize)) @@ -338,10 +338,10 @@ else: args = [space.wrap(self.typecode)] try: - dct = space.getattr(self, space.wrap('__dict__')) + w_dict = space.getattr(self, space.wrap('__dict__')) except OperationError: - dct = space.w_None - return space.newtuple([space.type(self), space.newtuple(args), dct]) + w_dict = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), w_dict]) def descr_copy(self, space): """ copy(array) diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py --- a/pypy/module/binascii/interp_base64.py +++ b/pypy/module/binascii/interp_base64.py @@ -71,7 +71,7 @@ if leftbits != 0: raise_Error(space, "Incorrect padding") - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -110,4 +110,4 @@ res.append(table_b2a_base64[(leftchar & 0xf) << 2]) res.append(PAD) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py --- a/pypy/module/binascii/interp_hexlify.py +++ b/pypy/module/binascii/interp_hexlify.py @@ -24,7 +24,7 @@ for c in data: res.append(_value2char(ord(c) >> 4)) res.append(_value2char(ord(c) & 0xf)) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -53,4 +53,4 @@ a = _char2value(space, hexstr[i]) b = _char2value(space, hexstr[i+1]) res.append(chr((a << 4) | b)) - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py --- a/pypy/module/binascii/interp_hqx.py +++ b/pypy/module/binascii/interp_hqx.py @@ -11,37 +11,37 @@ FAIL = 0x7d table_a2b_hqx = [ - #^@ ^A ^B ^C ^D ^E ^F ^G + #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #\b \t \n ^K ^L \r ^N ^O + #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, - #^P ^Q ^R ^S ^T ^U ^V ^W + #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ + #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, - # ! " # $ % & ' + # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - #( ) * + , - . / + #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, - #0 1 2 3 4 5 6 7 + #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, - #8 9 : ; < = > ? + #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, - #@ A B C D E F G + #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, - #H I J K L M N O + #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, - #P Q R S T U V W + #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, - #X Y Z [ \ ] ^ _ + #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, - #` a b c d e f g + #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, - #h i j k l m n o + #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, - #p q r s t u v w + #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, - #x y z { | } ~ ^? + #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, @@ -97,7 +97,7 @@ else: if pending_bits > 0: raise_Incomplete(space, 'String has incomplete number of bytes') - return space.newtuple([space.wrap(res.build()), space.wrap(done)]) + return space.newtuple([space.newbytes(res.build()), space.wrap(done)]) # ____________________________________________________________ @@ -128,7 +128,7 @@ if leftbits > 0: leftchar <<= (6 - leftbits) res.append(hqx_encoding[leftchar & 0x3f]) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -150,7 +150,7 @@ lastpushed = ord(c) else: if i == end: - raise_Incomplete(space, 'String ends with the RLE code \x90') + raise_Incomplete(space, 'String ends with the RLE code \\x90') count = ord(hexbin[i]) - 1 i += 1 if count < 0: @@ -158,9 +158,9 @@ lastpushed = 0x90 else: if lastpushed < 0: - raise_Error(space, 'String starts with the RLE code \x90') + raise_Error(space, 'String starts with the RLE code \\x90') res.append_multiple_char(chr(lastpushed), count) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -197,7 +197,7 @@ # string that rledecode_hqx() would expand back to 'data', there are # some programs somewhere that would start failing obscurely in rare # cases. - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py --- a/pypy/module/binascii/interp_qp.py +++ b/pypy/module/binascii/interp_qp.py @@ -56,7 +56,7 @@ if header and c == '_': c = ' ' odata.append(c) - return space.wrap(odata.build()) + return space.newbytes(odata.build()) # ____________________________________________________________ @@ -159,4 +159,4 @@ odata.append(c) inp += 1 - return space.wrap(odata.build()) + return space.newbytes(odata.build()) diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py --- a/pypy/module/binascii/interp_uu.py +++ b/pypy/module/binascii/interp_uu.py @@ -54,7 +54,7 @@ remaining = length - res.getlength() if remaining > 0: res.append_multiple_char('\x00', remaining) - return space.wrap(res.build()) + return space.newbytes(res.build()) # ____________________________________________________________ @@ -86,4 +86,4 @@ res.append(chr(0x20 + (C & 0x3F))) res.append('\n') - return space.wrap(res.build()) + return space.newbytes(res.build()) diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -557,7 +557,7 @@ datasize = len(data) if datasize == 0: - return self.space.wrap("") + return self.space.newbytes("") if not self.running: raise oefmt(self.space.w_ValueError, @@ -582,7 +582,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) def flush(self): if not self.running: @@ -602,7 +602,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", __doc__ = W_BZ2Compressor.__doc__, @@ -669,7 +669,7 @@ raise oefmt(self.space.w_EOFError, "end of stream was already found") if data == '': - return self.space.wrap('') + return self.space.newbytes('') in_bufsize = len(data) @@ -698,7 +698,7 @@ out.prepare_next_chunk() res = out.make_result_string() - return self.space.wrap(res) + return self.space.newbytes(res) W_BZ2Decompressor.typedef = TypeDef("BZ2Decompressor", diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -25,7 +25,7 @@ PyByteArrayObjectStruct = lltype.ForwardReference() PyByteArrayObject = lltype.Ptr(PyByteArrayObjectStruct) -PyByteArrayObjectFields = PyVarObjectFields +PyByteArrayObjectFields = PyVarObjectFields cpython_struct("PyByteArrayObject", PyByteArrayObjectFields, PyByteArrayObjectStruct) PyByteArray_Check, PyByteArray_CheckExact = build_type_checkers("ByteArray", "w_bytearray") @@ -46,9 +46,9 @@ """Create a new bytearray object from string and its length, len. On failure, NULL is returned.""" if char_p: - w_s = space.wrap(rffi.charpsize2str(char_p, length)) + w_s = space.newbytes(rffi.charpsize2str(char_p, length)) else: - w_s = space.wrap(length) + w_s = space.newint(length) w_buffer = space.call_function(space.w_bytearray, w_s) return make_ref(space, w_buffer) @@ -80,7 +80,7 @@ if space.isinstance_w(w_obj, space.w_bytearray): oldlen = space.len_w(w_obj) if newlen > oldlen: - space.call_method(w_obj, 'extend', space.wrap('\x00' * (newlen - oldlen))) + space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen))) elif oldlen > newlen: assert newlen >= 0 space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen)) diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -13,7 +13,7 @@ "PyCompilerFlags", (("cf_flags", rffi.INT),)) PyCompilerFlagsPtr = lltype.Ptr(PyCompilerFlags) -PyCF_MASK = (consts.CO_FUTURE_DIVISION | +PyCF_MASK = (consts.CO_FUTURE_DIVISION | consts.CO_FUTURE_ABSOLUTE_IMPORT | consts.CO_FUTURE_WITH_STATEMENT | consts.CO_FUTURE_PRINT_FUNCTION | @@ -94,7 +94,7 @@ Py_eval_input = 258 def compile_string(space, source, filename, start, flags=0): - w_source = space.wrap(source) + w_source = space.newbytes(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -227,4 +227,4 @@ cf.c_cf_flags = rffi.cast(rffi.INT, flags) return result - + diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -23,7 +23,7 @@ try: w_readline = space.getattr(w_obj, space.wrap('readline')) except OperationError: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "argument must be a file, or have a readline() method.") n = rffi.cast(lltype.Signed, n) @@ -41,7 +41,7 @@ On success, return a new file object that is opened on the file given by filename, with a file mode given by mode, where mode has the same semantics as the standard C routine fopen(). On failure, return NULL.""" - w_filename = space.wrap(rffi.charp2str(filename)) + w_filename = space.newbytes(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -7,7 +7,7 @@ space.wrap((2, 7)))): py.test.skip("unsupported before Python 2.7") - w_hello = space.wrap("hello") + w_hello = space.newbytes("hello") w_view = api.PyMemoryView_FromObject(w_hello) w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -20,7 +20,7 @@ assert api.PyFile_CheckExact(w_file) assert not api.PyFile_Check(space.wrap("text")) - space.call_method(w_file, "write", space.wrap("text")) + space.call_method(w_file, "write", space.newbytes("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -372,10 +372,10 @@ if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) + w_str = space.newbytes(rffi.charpsize2str(s, size)) w_encoding = space.wrap(rffi.charp2str(encoding)) - w_str = space.wrap(rffi.charpsize2str(s, size)) if errors: - w_errors = space.wrap(rffi.charp2str(errors)) + w_errors = space.newbytes(rffi.charp2str(errors)) else: w_errors = None return space.call_method(w_str, 'decode', w_encoding, w_errors) @@ -427,7 +427,7 @@ @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): """Create a Unicode object from an UTF-8 encoded null-terminated char buffer""" - w_str = space.wrap(rffi.charp2str(s)) + w_str = space.newbytes(rffi.charp2str(s)) return space.call_method(w_str, 'decode', space.wrap("utf-8")) @cpython_api([CONST_STRING, Py_ssize_t], PyObject, result_is_ll=True) @@ -495,7 +495,7 @@ encoded string s. Return NULL if an exception was raised by the codec. """ - w_s = space.wrap(rffi.charpsize2str(s, size)) + w_s = space.newbytes(rffi.charpsize2str(s, size)) if errors: w_errors = space.wrap(rffi.charp2str(errors)) else: diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -44,6 +44,7 @@ def str_w(self, space): return NonConstant("foobar") + identifier_w = bytes_w = str_w def unicode_w(self, space): return NonConstant(u"foobar") @@ -195,7 +196,7 @@ "NOT_RPYTHON" raise NotImplementedError - def wrapbytes(self, x): + def newbytes(self, x): return w_some_obj() def wrap(self, x): diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -578,7 +578,7 @@ def descr_str(self, space): if type(self) is W_BytesObject: return self - return wrapstr(space, self._value) + return W_BytesObject(self._value) def descr_hash(self, space): x = compute_hash(self._value) @@ -724,8 +724,8 @@ l = space.listview_bytes(w_list) if l is not None: if len(l) == 1: - return space.wrap(l[0]) - return space.wrap(self._val(space).join(l)) + return space.newbytes(l[0]) + return space.newbytes(self._val(space).join(l)) return self._StringMethods_descr_join(space, w_list) _StringMethods_descr_split = descr_split @@ -857,10 +857,6 @@ W_BytesObject.EMPTY = W_BytesObject('') -def wrapstr(space, s): - return W_BytesObject(s) - - W_BytesObject.typedef = TypeDef( "str", basestring_typedef, None, "read", __new__ = interp2app(W_BytesObject.descr_new), diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -16,7 +16,7 @@ from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.bufferobject import W_Buffer from pypy.objspace.std.bytearrayobject import W_BytearrayObject -from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject, wrapstr +from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject from pypy.objspace.std.complexobject import W_ComplexObject from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject from pypy.objspace.std.floatobject import W_FloatObject @@ -31,7 +31,7 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.tupleobject import W_AbstractTupleObject, W_TupleObject from pypy.objspace.std.typeobject import W_TypeObject, TypeCache -from pypy.objspace.std.unicodeobject import W_UnicodeObject, wrapunicode +from pypy.objspace.std.unicodeobject import W_UnicodeObject class StdObjSpace(ObjSpace): @@ -128,9 +128,6 @@ assert typedef is not None return self.fromcache(TypeCache).getorbuild(typedef) - def wrapbytes(self, x): - return wrapstr(self, x) - @specialize.argtype(1) def wrap(self, x): "Wraps the Python value 'x' into one of the wrapper classes." @@ -151,9 +148,9 @@ else: return self.newint(x) if isinstance(x, str): - return wrapstr(self, x) + return self.newbytes(x) if isinstance(x, unicode): - return wrapunicode(self, x) + return self.newunicode(x) if isinstance(x, float): return W_FloatObject(x) if isinstance(x, W_Root): @@ -323,6 +320,12 @@ def newbuffer(self, w_obj): return W_Buffer(w_obj) + def newbytes(self, s): + return W_BytesObject(s) + + def newunicode(self, uni): + return W_UnicodeObject(uni) + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py --- a/pypy/objspace/std/test/test_bytesobject.py +++ b/pypy/objspace/std/test/test_bytesobject.py @@ -3,31 +3,31 @@ def teardown_method(self, method): pass - def test_str_w(self): - assert self.space.str_w(self.space.wrap("foo")) == "foo" + def test_bytes_w(self): + assert self.space.bytes_w(self.space.newbytes("foo")) == "foo" def test_equality(self): - w = self.space.wrap + w = self.space.newbytes assert self.space.eq_w(w('abc'), w('abc')) assert not self.space.eq_w(w('abc'), w('def')) def test_order_cmp(self): space = self.space - w = space.wrap + w = space.newbytes assert self.space.is_true(space.lt(w('a'), w('b'))) assert self.space.is_true(space.lt(w('a'), w('ab'))) assert self.space.is_true(space.le(w('a'), w('a'))) assert self.space.is_true(space.gt(w('a'), w(''))) def test_truth(self): - w = self.space.wrap + w = self.space.newbytes assert self.space.is_true(w('non-empty')) assert not self.space.is_true(w('')) def test_getitem(self): space = self.space w = space.wrap - w_str = w('abc') + w_str = space.newbytes('abc') assert self.space.eq_w(space.getitem(w_str, w(0)), w('a')) assert self.space.eq_w(space.getitem(w_str, w(-1)), w('c')) self.space.raises_w(space.w_IndexError, @@ -38,25 +38,26 @@ def test_slice(self): space = self.space w = space.wrap - w_str = w('abc') + wb = space.newbytes + w_str = wb('abc') w_slice = space.newslice(w(0), w(0), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('')) w_slice = space.newslice(w(0), w(1), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('a')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('a')) w_slice = space.newslice(w(0), w(10), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc')) w_slice = space.newslice(space.w_None, space.w_None, space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc')) w_slice = space.newslice(space.w_None, w(-1), space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('ab')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('ab')) w_slice = space.newslice(w(-1), space.w_None, space.w_None) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('c')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('c')) def test_extended_slice(self): space = self.space @@ -66,23 +67,24 @@ return w_None = space.w_None w = space.wrap - w_str = w('hello') + wb = space.newbytes + w_str = wb('hello') w_slice = space.newslice(w_None, w_None, w(1)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('hello')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hello')) w_slice = space.newslice(w_None, w_None, w(-1)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('olleh')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('olleh')) w_slice = space.newslice(w_None, w_None, w(2)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('hlo')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hlo')) w_slice = space.newslice(w(1), w_None, w(2)) - assert self.space.eq_w(space.getitem(w_str, w_slice), w('el')) + assert self.space.eq_w(space.getitem(w_str, w_slice), wb('el')) def test_listview_bytes(self): - w_str = self.space.wrap('abcd') - assert self.space.listview_bytes(w_str) == list("abcd") + w_bytes = self.space.newbytes('abcd') + assert self.space.listview_bytes(w_bytes) == list("abcd") class AppTestBytesObject: @@ -110,28 +112,29 @@ assert str(exc_info.value) == expected def test_split(self): - assert "".split() == [] - assert "".split('x') == [''] - assert " ".split() == [] - assert "a".split() == ['a'] + assert b"".split() == [] + assert b"".split(b'x') == [b''] + assert b" ".split() == [] + assert b"a".split() == [b'a'] assert "a".split("a", 1) == ['', ''] - assert " ".split(" ", 1) == ['', ''] - assert "aa".split("a", 2) == ['', '', ''] - assert " a ".split() == ['a'] - assert "a b c".split() == ['a','b','c'] - assert 'this is the split function'.split() == ['this', 'is', 'the', 'split', 'function'] - assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd'] - assert 'a|b|c|d'.split('|', 2) == ['a', 'b', 'c|d'] - assert 'a b c d'.split(None, 1) == ['a', 'b c d'] - assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d'] - assert 'a b c d'.split(None, 3) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.split(None, 4) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.split(None, 0) == ['a b c d'] - assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d'] - assert 'a b c d '.split() == ['a', 'b', 'c', 'd'] - assert 'a//b//c//d'.split('//') == ['a', 'b', 'c', 'd'] - assert 'endcase test'.split('test') == ['endcase ', ''] - raises(ValueError, 'abc'.split, '') + assert b" ".split(b" ", 1) == [b'', b''] + assert b"aa".split(b"a", 2) == [b'', b'', b''] + assert b" a ".split() == [b'a'] + assert b"a b c".split() == [b'a',b'b',b'c'] + assert b'this is the split function'.split() == [ + b'this', b'is', b'the', b'split', b'function'] + assert b'a|b|c|d'.split(b'|') == [b'a', b'b', b'c', b'd'] + assert b'a|b|c|d'.split(b'|', 2) == [b'a', b'b', b'c|d'] + assert b'a b c d'.split(None, 1) == [b'a', b'b c d'] + assert b'a b c d'.split(None, 2) == [b'a', b'b', b'c d'] + assert b'a b c d'.split(None, 3) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.split(None, 4) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.split(None, 0) == [b'a b c d'] + assert b'a b c d'.split(None, 2) == [b'a', b'b', b'c d'] + assert b'a b c d '.split() == [b'a', b'b', b'c', b'd'] + assert b'a//b//c//d'.split(b'//') == [b'a', b'b', b'c', b'd'] + assert b'endcase test'.split(b'test') == [b'endcase ', b''] + raises(ValueError, b'abc'.split, b'') def test_rsplit(self): assert "".rsplit() == [] @@ -141,100 +144,100 @@ assert " ".rsplit(" ", 1) == ['', ''] assert "aa".rsplit("a", 2) == ['', '', ''] assert " a ".rsplit() == ['a'] - assert "a b c".rsplit() == ['a','b','c'] + assert b"a b c".rsplit() == [b'a',b'b',b'c'] assert 'this is the rsplit function'.rsplit() == ['this', 'is', 'the', 'rsplit', 'function'] - assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd'] - assert 'a|b|c|d'.rsplit('|', 2) == ['a|b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 1) == ['a b c', 'd'] - assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 3) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 4) == ['a', 'b', 'c', 'd'] - assert 'a b c d'.rsplit(None, 0) == ['a b c d'] - assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd'] - assert 'a b c d '.rsplit() == ['a', 'b', 'c', 'd'] - assert 'a//b//c//d'.rsplit('//') == ['a', 'b', 'c', 'd'] - assert 'endcase test'.rsplit('test') == ['endcase ', ''] - raises(ValueError, 'abc'.rsplit, '') + assert b'a|b|c|d'.rsplit(b'|') == [b'a', b'b', b'c', b'd'] + assert b'a|b|c|d'.rsplit(b'|', 2) == [b'a|b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 1) == [b'a b c', b'd'] + assert b'a b c d'.rsplit(None, 2) == [b'a b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 3) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 4) == [b'a', b'b', b'c', b'd'] + assert b'a b c d'.rsplit(None, 0) == [b'a b c d'] + assert b'a b c d'.rsplit(None, 2) == [b'a b', b'c', b'd'] + assert b'a b c d '.rsplit() == [b'a', b'b', b'c', b'd'] + assert b'a//b//c//d'.rsplit(b'//') == [b'a', b'b', b'c', b'd'] + assert b'endcase test'.rsplit(b'test') == [b'endcase ', b''] + raises(ValueError, b'abc'.rsplit, b'') def test_split_splitchar(self): assert "/a/b/c".split('/') == ['','a','b','c'] def test_title(self): - assert "brown fox".title() == "Brown Fox" - assert "!brown fox".title() == "!Brown Fox" - assert "bROWN fOX".title() == "Brown Fox" - assert "Brown Fox".title() == "Brown Fox" - assert "bro!wn fox".title() == "Bro!Wn Fox" + assert b"brown fox".title() == b"Brown Fox" + assert b"!brown fox".title() == b"!Brown Fox" + assert b"bROWN fOX".title() == b"Brown Fox" + assert b"Brown Fox".title() == b"Brown Fox" + assert b"bro!wn fox".title() == b"Bro!Wn Fox" def test_istitle(self): - assert "".istitle() == False - assert "!".istitle() == False - assert "!!".istitle() == False - assert "brown fox".istitle() == False - assert "!brown fox".istitle() == False - assert "bROWN fOX".istitle() == False - assert "Brown Fox".istitle() == True - assert "bro!wn fox".istitle() == False - assert "Bro!wn fox".istitle() == False - assert "!brown Fox".istitle() == False - assert "!Brown Fox".istitle() == True - assert "Brow&&&&N Fox".istitle() == True - assert "!Brow&&&&n Fox".istitle() == False + assert b"".istitle() == False + assert b"!".istitle() == False + assert b"!!".istitle() == False + assert b"brown fox".istitle() == False + assert b"!brown fox".istitle() == False + assert b"bROWN fOX".istitle() == False + assert b"Brown Fox".istitle() == True + assert b"bro!wn fox".istitle() == False + assert b"Bro!wn fox".istitle() == False + assert b"!brown Fox".istitle() == False + assert b"!Brown Fox".istitle() == True + assert b"Brow&&&&N Fox".istitle() == True + assert b"!Brow&&&&n Fox".istitle() == False def test_capitalize(self): - assert "brown fox".capitalize() == "Brown fox" - assert ' hello '.capitalize() == ' hello ' - assert 'Hello '.capitalize() == 'Hello ' - assert 'hello '.capitalize() == 'Hello ' - assert 'aaaa'.capitalize() == 'Aaaa' - assert 'AaAa'.capitalize() == 'Aaaa' + assert b"brown fox".capitalize() == b"Brown fox" + assert b' hello '.capitalize() == b' hello ' + assert b'Hello '.capitalize() == b'Hello ' + assert b'hello '.capitalize() == b'Hello ' + assert b'aaaa'.capitalize() == b'Aaaa' + assert b'AaAa'.capitalize() == b'Aaaa' def test_rjust(self): - s = "abc" + s = b"abc" assert s.rjust(2) == s assert s.rjust(3) == s - assert s.rjust(4) == " " + s - assert s.rjust(5) == " " + s - assert 'abc'.rjust(10) == ' abc' - assert 'abc'.rjust(6) == ' abc' - assert 'abc'.rjust(3) == 'abc' - assert 'abc'.rjust(2) == 'abc' - assert 'abc'.rjust(5, '*') == '**abc' # Python 2.4 + assert s.rjust(4) == b" " + s + assert s.rjust(5) == b" " + s + assert b'abc'.rjust(10) == b' abc' + assert b'abc'.rjust(6) == b' abc' + assert b'abc'.rjust(3) == b'abc' + assert b'abc'.rjust(2) == b'abc' + assert b'abc'.rjust(5, b'*') == b'**abc' # Python 2.4 raises(TypeError, 'abc'.rjust, 5, 'xx') def test_ljust(self): - s = "abc" + s = b"abc" assert s.ljust(2) == s assert s.ljust(3) == s - assert s.ljust(4) == s + " " - assert s.ljust(5) == s + " " - assert 'abc'.ljust(10) == 'abc ' - assert 'abc'.ljust(6) == 'abc ' - assert 'abc'.ljust(3) == 'abc' - assert 'abc'.ljust(2) == 'abc' - assert 'abc'.ljust(5, '*') == 'abc**' # Python 2.4 + assert s.ljust(4) == s + b" " + assert s.ljust(5) == s + b" " + assert b'abc'.ljust(10) == b'abc ' + assert b'abc'.ljust(6) == b'abc ' + assert b'abc'.ljust(3) == b'abc' + assert b'abc'.ljust(2) == b'abc' + assert b'abc'.ljust(5, b'*') == b'abc**' # Python 2.4 raises(TypeError, 'abc'.ljust, 6, '') def test_replace(self): - assert 'one!two!three!'.replace('!', '@', 1) == 'one at two!three!' - assert 'one!two!three!'.replace('!', '') == 'onetwothree' - assert 'one!two!three!'.replace('!', '@', 2) == 'one at two@three!' - assert 'one!two!three!'.replace('!', '@', 3) == 'one at two@three@' - assert 'one!two!three!'.replace('!', '@', 4) == 'one at two@three@' - assert 'one!two!three!'.replace('!', '@', 0) == 'one!two!three!' - assert 'one!two!three!'.replace('!', '@') == 'one at two@three@' - assert 'one!two!three!'.replace('x', '@') == 'one!two!three!' - assert 'one!two!three!'.replace('x', '@', 2) == 'one!two!three!' - assert 'abc'.replace('', '-') == '-a-b-c-' - assert 'abc'.replace('', '-', 3) == '-a-b-c' - assert 'abc'.replace('', '-', 0) == 'abc' - assert ''.replace('', '') == '' - assert ''.replace('', 'a') == 'a' - assert 'abc'.replace('ab', '--', 0) == 'abc' - assert 'abc'.replace('xy', '--') == 'abc' - assert '123'.replace('123', '') == '' - assert '123123'.replace('123', '') == '' - assert '123x123'.replace('123', '') == 'x' + assert b'one!two!three!'.replace(b'!', b'@', 1) == b'one at two!three!' + assert b'one!two!three!'.replace(b'!', b'') == b'onetwothree' + assert b'one!two!three!'.replace(b'!', b'@', 2) == b'one at two@three!' + assert b'one!two!three!'.replace(b'!', b'@', 3) == b'one at two@three@' + assert b'one!two!three!'.replace(b'!', b'@', 4) == b'one at two@three@' + assert b'one!two!three!'.replace(b'!', b'@', 0) == b'one!two!three!' + assert b'one!two!three!'.replace(b'!', b'@') == b'one at two@three@' + assert b'one!two!three!'.replace(b'x', b'@') == b'one!two!three!' + assert b'one!two!three!'.replace(b'x', b'@', 2) == b'one!two!three!' + assert b'abc'.replace(b'', b'-') == b'-a-b-c-' + assert b'abc'.replace(b'', b'-', 3) == b'-a-b-c' + assert b'abc'.replace(b'', b'-', 0) == b'abc' + assert b''.replace(b'', b'') == b'' + assert b''.replace(b'', b'a') == b'a' + assert b'abc'.replace(b'ab', b'--', 0) == b'abc' + assert b'abc'.replace(b'xy', b'--') == b'abc' + assert b'123'.replace(b'123', b'') == b'' + assert b'123123'.replace(b'123', b'') == b'' + assert b'123x123'.replace(b'123', b'') == b'x' def test_replace_buffer(self): assert 'one'.replace(buffer('o'), buffer('n'), 1) == 'nne' @@ -245,23 +248,23 @@ assert s.strip() == "a b" assert s.rstrip() == " a b" assert s.lstrip() == "a b " - assert 'xyzzyhelloxyzzy'.strip('xyz') == 'hello' - assert 'xyzzyhelloxyzzy'.lstrip('xyz') == 'helloxyzzy' - assert 'xyzzyhelloxyzzy'.rstrip('xyz') == 'xyzzyhello' + assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello' + assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy' + assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello' def test_zfill(self): - assert '123'.zfill(2) == '123' - assert '123'.zfill(3) == '123' - assert '123'.zfill(4) == '0123' - assert '+123'.zfill(3) == '+123' - assert '+123'.zfill(4) == '+123' - assert '+123'.zfill(5) == '+0123' - assert '-123'.zfill(3) == '-123' - assert '-123'.zfill(4) == '-123' - assert '-123'.zfill(5) == '-0123' - assert ''.zfill(3) == '000' - assert '34'.zfill(1) == '34' - assert '34'.zfill(4) == '0034' + assert b'123'.zfill(2) == b'123' + assert b'123'.zfill(3) == b'123' + assert b'123'.zfill(4) == b'0123' + assert b'+123'.zfill(3) == b'+123' + assert b'+123'.zfill(4) == b'+123' + assert b'+123'.zfill(5) == b'+0123' + assert b'-123'.zfill(3) == b'-123' + assert b'-123'.zfill(4) == b'-123' + assert b'-123'.zfill(5) == b'-0123' + assert b''.zfill(3) == b'000' + assert b'34'.zfill(1) == b'34' + assert b'34'.zfill(4) == b'0034' def test_center(self): s="a b" @@ -275,251 +278,251 @@ assert s.center(7) == " a b " assert s.center(8) == " a b " assert s.center(9) == " a b " - assert 'abc'.center(10) == ' abc ' - assert 'abc'.center(6) == ' abc ' - assert 'abc'.center(3) == 'abc' - assert 'abc'.center(2) == 'abc' - assert 'abc'.center(5, '*') == '*abc*' # Python 2.4 - raises(TypeError, 'abc'.center, 4, 'cba') - assert ' abc'.center(7) == ' abc ' + assert b'abc'.center(10) == b' abc ' + assert b'abc'.center(6) == b' abc ' + assert b'abc'.center(3) == b'abc' + assert b'abc'.center(2) == b'abc' + assert b'abc'.center(5, b'*') == b'*abc*' # Python 2.4 + raises(TypeError, b'abc'.center, 4, b'cba') + assert b' abc'.center(7) == b' abc ' def test_count(self): - assert "".count("x") ==0 - assert "".count("") ==1 - assert "Python".count("") ==7 - assert "ab aaba".count("ab") ==2 - assert 'aaa'.count('a') == 3 - assert 'aaa'.count('b') == 0 - assert 'aaa'.count('a', -1) == 1 - assert 'aaa'.count('a', -10) == 3 - assert 'aaa'.count('a', 0, -1) == 2 - assert 'aaa'.count('a', 0, -10) == 0 - assert 'ababa'.count('aba') == 1 + assert b"".count(b"x") ==0 + assert b"".count(b"") ==1 + assert b"Python".count(b"") ==7 + assert b"ab aaba".count(b"ab") ==2 + assert b'aaa'.count(b'a') == 3 + assert b'aaa'.count(b'b') == 0 + assert b'aaa'.count(b'a', -1) == 1 + assert b'aaa'.count(b'a', -10) == 3 + assert b'aaa'.count(b'a', 0, -1) == 2 + assert b'aaa'.count(b'a', 0, -10) == 0 + assert b'ababa'.count(b'aba') == 1 def test_startswith(self): - assert 'ab'.startswith('ab') is True - assert 'ab'.startswith('a') is True - assert 'ab'.startswith('') is True - assert 'x'.startswith('a') is False - assert 'x'.startswith('x') is True - assert ''.startswith('') is True - assert ''.startswith('a') is False - assert 'x'.startswith('xx') is False - assert 'y'.startswith('xx') is False + assert b'ab'.startswith(b'ab') is True + assert b'ab'.startswith(b'a') is True + assert b'ab'.startswith(b'') is True + assert b'x'.startswith(b'a') is False + assert b'x'.startswith(b'x') is True + assert b''.startswith(b'') is True + assert b''.startswith(b'a') is False + assert b'x'.startswith(b'xx') is False + assert b'y'.startswith(b'xx') is False def test_startswith_more(self): - assert 'ab'.startswith('a', 0) is True - assert 'ab'.startswith('a', 1) is False - assert 'ab'.startswith('b', 1) is True - assert 'abc'.startswith('bc', 1, 2) is False - assert 'abc'.startswith('c', -1, 4) is True + assert b'ab'.startswith(b'a', 0) is True + assert b'ab'.startswith(b'a', 1) is False + assert b'ab'.startswith(b'b', 1) is True + assert b'abc'.startswith(b'bc', 1, 2) is False + assert b'abc'.startswith(b'c', -1, 4) is True def test_startswith_too_large(self): - assert 'ab'.startswith('b', 1) is True - assert 'ab'.startswith('', 2) is True - assert 'ab'.startswith('', 3) is False - assert 'ab'.endswith('b', 1) is True - assert 'ab'.endswith('', 2) is True - assert 'ab'.endswith('', 3) is False + assert b'ab'.startswith(b'b', 1) is True + assert b'ab'.startswith(b'', 2) is True + assert b'ab'.startswith(b'', 3) is False + assert b'ab'.endswith(b'b', 1) is True + assert b'ab'.endswith(b'', 2) is True + assert b'ab'.endswith(b'', 3) is False def test_startswith_tuples(self): - assert 'hello'.startswith(('he', 'ha')) - assert not 'hello'.startswith(('lo', 'llo')) - assert 'hello'.startswith(('hellox', 'hello')) - assert not 'hello'.startswith(()) - assert 'helloworld'.startswith(('hellowo', 'rld', 'lowo'), 3) - assert not 'helloworld'.startswith(('hellowo', 'ello', 'rld'), 3) - assert 'hello'.startswith(('lo', 'he'), 0, -1) - assert not 'hello'.startswith(('he', 'hel'), 0, 1) - assert 'hello'.startswith(('he', 'hel'), 0, 2) - raises(TypeError, 'hello'.startswith, (42,)) + assert b'hello'.startswith((b'he', b'ha')) + assert not b'hello'.startswith((b'lo', b'llo')) + assert b'hello'.startswith((b'hellox', b'hello')) + assert not b'hello'.startswith(()) + assert b'helloworld'.startswith((b'hellowo', b'rld', b'lowo'), 3) + assert not b'helloworld'.startswith((b'hellowo', b'ello', b'rld'), 3) + assert b'hello'.startswith((b'lo', b'he'), 0, -1) + assert not b'hello'.startswith((b'he', b'hel'), 0, 1) + assert b'hello'.startswith((b'he', b'hel'), 0, 2) + raises(TypeError, b'hello'.startswith, (42,)) def test_endswith(self): - assert 'ab'.endswith('ab') is True - assert 'ab'.endswith('b') is True - assert 'ab'.endswith('') is True - assert 'x'.endswith('a') is False - assert 'x'.endswith('x') is True - assert ''.endswith('') is True - assert ''.endswith('a') is False - assert 'x'.endswith('xx') is False - assert 'y'.endswith('xx') is False + assert b'ab'.endswith(b'ab') is True + assert b'ab'.endswith(b'b') is True + assert b'ab'.endswith(b'') is True + assert b'x'.endswith(b'a') is False + assert b'x'.endswith(b'x') is True + assert b''.endswith(b'') is True + assert b''.endswith(b'a') is False + assert b'x'.endswith(b'xx') is False + assert b'y'.endswith(b'xx') is False def test_endswith_more(self): - assert 'abc'.endswith('ab', 0, 2) is True - assert 'abc'.endswith('bc', 1) is True - assert 'abc'.endswith('bc', 2) is False - assert 'abc'.endswith('b', -3, -1) is True + assert b'abc'.endswith(b'ab', 0, 2) is True + assert b'abc'.endswith(b'bc', 1) is True + assert b'abc'.endswith(b'bc', 2) is False + assert b'abc'.endswith(b'b', -3, -1) is True def test_endswith_tuple(self): - assert not 'hello'.endswith(('he', 'ha')) - assert 'hello'.endswith(('lo', 'llo')) - assert 'hello'.endswith(('hellox', 'hello')) - assert not 'hello'.endswith(()) - assert 'helloworld'.endswith(('hellowo', 'rld', 'lowo'), 3) - assert not 'helloworld'.endswith(('hellowo', 'ello', 'rld'), 3, -1) - assert 'hello'.endswith(('hell', 'ell'), 0, -1) - assert not 'hello'.endswith(('he', 'hel'), 0, 1) - assert 'hello'.endswith(('he', 'hell'), 0, 4) - raises(TypeError, 'hello'.endswith, (42,)) + assert not b'hello'.endswith((b'he', b'ha')) + assert b'hello'.endswith((b'lo', b'llo')) + assert b'hello'.endswith((b'hellox', b'hello')) + assert not b'hello'.endswith(()) + assert b'helloworld'.endswith((b'hellowo', b'rld', b'lowo'), 3) + assert not b'helloworld'.endswith((b'hellowo', b'ello', b'rld'), 3, -1) + assert b'hello'.endswith((b'hell', b'ell'), 0, -1) + assert not b'hello'.endswith((b'he', b'hel'), 0, 1) + assert b'hello'.endswith((b'he', b'hell'), 0, 4) + raises(TypeError, b'hello'.endswith, (42,)) def test_expandtabs(self): import sys From pypy.commits at gmail.com Thu Jun 30 10:37:12 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 30 Jun 2016 07:37:12 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: add a test to ensure the correct assembly for a xx3 form Message-ID: <57752e98.4d9a1c0a.c9c13.ffffeec8@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85474:2374b5acaee5 Date: 2016-06-30 14:14 +0200 http://bitbucket.org/pypy/pypy/changeset/2374b5acaee5/ Log: add a test to ensure the correct assembly for a xx3 form diff --git a/rpython/jit/backend/ppc/test/test_vector_instr.py b/rpython/jit/backend/ppc/test/test_vector_instr.py --- a/rpython/jit/backend/ppc/test/test_vector_instr.py +++ b/rpython/jit/backend/ppc/test/test_vector_instr.py @@ -61,7 +61,7 @@ @vec_asmtest(memory=[(16, signed, [0,0])]) def test_unaligned_load(self, a, mem): a.load_imm(r15, mem) - a.lxvd2x(vr0.value, 0, r15.value) + a.lxvd2x(0, 0, r15.value) a.blr() return [ (0, signed, mem), (0, signed, mem+8) ] @@ -69,8 +69,27 @@ def test_unaligned_load_and_store(self, a, mem_l, mem_t): a.load_imm(r15, mem_l) a.load_imm(r14, mem_t) - a.lxvd2x(vr0.value, 0, r15.value) - a.stxvd2x(vr0.value, 0, r14.value) + a.lxvd2x(0, 0, r15.value) + a.stxvd2x(0, 0, r14.value) a.blr() return [ (1, signed, mem_t), (2, signed, mem_t+8) ] + def test_xx3_instr(self): + a = PPCBuilder() + def assign_to_self(v): + self.last_value = v + a.emit = assign_to_self + + a.xxspltdl(32, 32, 32) + # tttttaaaaabbbbb abt + assert hex(int(self.last_value)) == hex(0b11110000000000000000000001010111) + a.xxspltdl(32, 2, 2) + # tttttaaaaabbbbb abt + assert hex(int(self.last_value)) == hex(0b11110000000000100001000001010001) + a.xxspltdl(0, 63, 0) + # tttttaaaaabbbbb abt + assert hex(int(self.last_value)) == hex(0b11110000000111110000000001010100) + a.xxspltdl(0, 0, 63) + # tttttaaaaabbbbb abt + assert hex(int(self.last_value)) == hex(0b11110000000000001111100001010010) + From pypy.commits at gmail.com Thu Jun 30 10:37:14 2016 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 30 Jun 2016 07:37:14 -0700 (PDT) Subject: [pypy-commit] pypy ppc-vsx-support: provide vec_expand_i implementation Message-ID: <57752e9a.03c31c0a.e93e0.23c5@mx.google.com> Author: Richard Plangger Branch: ppc-vsx-support Changeset: r85475:c2a7f4349490 Date: 2016-06-30 16:36 +0200 http://bitbucket.org/pypy/pypy/changeset/c2a7f4349490/ Log: provide vec_expand_i implementation diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py --- a/rpython/jit/backend/ppc/codebuilder.py +++ b/rpython/jit/backend/ppc/codebuilder.py @@ -706,6 +706,11 @@ vsel = VA(4, XO10=42) vspltisb = VXI(4, XO8=780) + VX_splat = Form("ivrT", "ivrB", "ivrA", "XO8") + vspltb = VX_splat(4, XO8=524) + vsplth = VX_splat(4, XO8=588) + vspltw = VX_splat(4, XO8=652) + diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py --- a/rpython/jit/backend/ppc/vector_ext.py +++ b/rpython/jit/backend/ppc/vector_ext.py @@ -481,35 +481,35 @@ elif size == 8: # splat the low of src to both slots in res src = srcloc.value - #import pdb; pdb.set_trace() self.mc.xxspltdl(res, src, src) else: notimplemented("[ppc/assembler] vec expand in this combination not supported") def emit_vec_expand_i(self, op, arglocs, regalloc): - notimplemented("[vec expand i]") - srcloc, sizeloc = arglocs - if not isinstance(srcloc, RegLoc): - self.mov(srcloc, X86_64_SCRATCH_REG) - srcloc = X86_64_SCRATCH_REG - assert not srcloc.is_xmm - size = sizeloc.value + res, l0, off = arglocs + size = op.bytesize + + self.mc.load_imm(r.SCRATCH2, off.value) + self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value) if size == 1: - self.mc.PINSRB_xri(resloc.value, srcloc.value, 0) - self.mc.PSHUFB(resloc, heap(self.expand_byte_mask_addr)) + if IS_BIG_ENDIAN: + self.mc.vspltb(res.value, res.value, 0b0000) + else: + self.mc.vspltb(res.value, res.value, 0b1111) elif size == 2: - self.mc.PINSRW_xri(resloc.value, srcloc.value, 0) - self.mc.PINSRW_xri(resloc.value, srcloc.value, 4) - self.mc.PSHUFLW_xxi(resloc.value, resloc.value, 0) - self.mc.PSHUFHW_xxi(resloc.value, resloc.value, 0) + if IS_BIG_ENDIAN: + self.mc.vsplth(res.value, res.value, 0b000) + else: + self.mc.vsplth(res.value, res.value, 0b111) elif size == 4: - self.mc.PINSRD_xri(resloc.value, srcloc.value, 0) - self.mc.PSHUFD_xxi(resloc.value, resloc.value, 0) + if IS_BIG_ENDIAN: + self.mc.vspltw(res.value, res.value, 0b00) + else: + self.mc.vspltw(res.value, res.value, 0b11) elif size == 8: - self.mc.PINSRQ_xri(resloc.value, srcloc.value, 0) - self.mc.PINSRQ_xri(resloc.value, srcloc.value, 1) + pass else: - raise AssertionError("cannot handle size %d (int expand)" % (size,)) + notimplemented("[expand int size not impl]") #def genop_vec_pack_i(self, op, arglocs, regalloc): # resultloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs @@ -811,7 +811,20 @@ res = self.force_allocate_vector_reg(op) return [res, l0] - prepare_vec_expand_i = prepare_vec_expand_f + def prepare_vec_expand_i(self, op): + arg = op.getarg(0) + mc = self.assembler.mc + if arg.is_constant(): + l0 = self.rm.get_scratch_reg() + mc.load_imm(l0, arg.value) + else: + l0 = self.ensure_reg(arg) + mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET) + size = op.bytesize + if size == 8: + mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8) + res = self.force_allocate_vector_reg(op) + return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)] def prepare_vec_int_is_true(self, op): arg = op.getarg(0) diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py --- a/rpython/jit/metainterp/optimizeopt/schedule.py +++ b/rpython/jit/metainterp/optimizeopt/schedule.py @@ -755,7 +755,7 @@ node.pack = self node.pack_position = i - def split(self, packlist, vec_reg_size): + def split(self, packlist, vec_reg_size, vector_ext): """ Combination phase creates the biggest packs that are possible. In this step the pack is reduced in size to fit into an vector register. @@ -764,7 +764,7 @@ pack = self while pack.pack_load(vec_reg_size) > Pack.FULL: pack.clear() - oplist, newoplist = pack.slice_operations(vec_reg_size) + oplist, newoplist = pack.slice_operations(vec_reg_size, vector_ext) pack.operations = oplist pack.update_pack_of_nodes() if not pack.leftmost().is_typecast(): @@ -782,13 +782,13 @@ break pack.update_pack_of_nodes() - def opcount_filling_vector_register(self, vec_reg_size): + def opcount_filling_vector_register(self, vec_reg_size, vector_ext): left = self.leftmost() - oprestrict = trans.get(left) + oprestrict = vector_ext.get_operation_restriction(left) return oprestrict.opcount_filling_vector_register(left, vec_reg_size) - def slice_operations(self, vec_reg_size): - count = self.opcount_filling_vector_register(vec_reg_size) + def slice_operations(self, vec_reg_size, vector_ext): + count = self.opcount_filling_vector_register(vec_reg_size, vector_ext) assert count > 0 newoplist = self.operations[count:] oplist = self.operations[:count] diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py --- a/rpython/jit/metainterp/optimizeopt/vector.py +++ b/rpython/jit/metainterp/optimizeopt/vector.py @@ -451,7 +451,7 @@ if len_before == len(self.packset.packs): break - self.packset.split_overloaded_packs() + self.packset.split_overloaded_packs(self.cpu.vector_ext) if not we_are_translated(): # some test cases check the accumulation variables @@ -814,12 +814,12 @@ state.setvector_of_box(seed, 0, vecop) # prevent it from expansion state.renamer.start_renaming(seed, vecop) - def split_overloaded_packs(self): + def split_overloaded_packs(self, vector_ext): newpacks = [] for i,pack in enumerate(self.packs): load = pack.pack_load(self.vec_reg_size) if load > Pack.FULL: - pack.split(newpacks, self.vec_reg_size) + pack.split(newpacks, self.vec_reg_size, vector_ext) continue if load < Pack.FULL: for op in pack.operations: diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py --- a/rpython/jit/metainterp/test/test_vector.py +++ b/rpython/jit/metainterp/test/test_vector.py @@ -430,21 +430,24 @@ res = self.meta_interp(f, [60], vec_all=True) assert res == f(60) == 34.5 - def test_variable_expand(self): + @py.test.mark.parametrize('type,value', [(rffi.DOUBLE, 58.4547), + (lltype.Signed, 2300000), (rffi.INT, 4321), + (rffi.SHORT, 9922), (rffi.SIGNEDCHAR, -127)]) + def test_variable_expand(self, type, value): myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True) - T = lltype.Array(rffi.DOUBLE, hints={'nolength': True}) + T = lltype.Array(type, hints={'nolength': True}) def f(d,variable): va = lltype.malloc(T, d, flavor='raw', zero=True) i = 0 while i < d: myjitdriver.jit_merge_point() - va[i] = va[i] + variable + va[i] = rffi.cast(type, variable) i += 1 val = va[d//2] lltype.free(va, flavor='raw') return val - res = self.meta_interp(f, [60,58.4547]) - assert res == f(60,58.4547) == 58.4547 + res = self.meta_interp(f, [60,value]) + assert res == f(60,value) == value @py.test.mark.parametrize('vec,vec_all',[(False,True),(True,False),(True,True),(False,False)]) def test_accum(self, vec, vec_all): From pypy.commits at gmail.com Thu Jun 30 17:01:58 2016 From: pypy.commits at gmail.com (arigo) Date: Thu, 30 Jun 2016 14:01:58 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <577588c6.a758c20a.16722.ffffc8de@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r763:2d9c25ad9cdf Date: 2016-06-30 23:03 +0200 http://bitbucket.org/pypy/pypy.org/changeset/2d9c25ad9cdf/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $64615 of $105000 (61.5%) + $64624 of $105000 (61.5%)
    @@ -23,7 +23,7 @@
  • From pypy.commits at gmail.com Thu Jun 30 18:55:22 2016 From: pypy.commits at gmail.com (wlav) Date: Thu, 30 Jun 2016 15:55:22 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: from Aditi: test_cppyy.py now succeeds Message-ID: <5775a35a.46461c0a.9bab3.2d40@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85477:107dd38e12c9 Date: 2016-06-30 15:52 -0700 http://bitbucket.org/pypy/pypy/changeset/107dd38e12c9/ Log: from Aditi: test_cppyy.py now succeeds diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -255,6 +255,8 @@ def convert_argument(self, space, w_obj, address, call_local): x = rffi.cast(self.c_ptrtype, address) x[0] = self._unwrap_object(space, w_obj) + ba = rffi.cast(rffi.CCHARP, address) + ba[capi.c_function_arg_typeoffset(space)] = self.typecode class FloatTypeConverterMixin(NumericTypeConverterMixin): _mixin_ = True @@ -337,7 +339,7 @@ _immutable_fields_ = ['libffitype', 'typecode'] libffitype = jit_libffi.types.pointer - typecode = 'F' + typecode = 'f' def convert_argument_libffi(self, space, w_obj, address, call_local): from pypy.module.cppyy.interp_cppyy import FastCallNotPossible @@ -356,7 +358,7 @@ _immutable_fields_ = ['libffitype', 'typecode'] libffitype = jit_libffi.types.pointer - typecode = 'D' + typecode = 'd' class CStringConverter(TypeConverter): @@ -713,17 +715,18 @@ "NOT_RPYTHON" # signed types (use strtoll in setting of default in __init__) type_info = ( - (rffi.SHORT, ("short", "short int")), - (rffi.INT, ("int",)), + (rffi.SHORT, ("short", "short int"), 'h'), + (rffi.INT, ("int",), 'i'), ) # constref converters exist only b/c the stubs take constref by value, whereas # libffi takes them by pointer (hence it needs the fast-path in testing); note # that this is list is not complete, as some classes are specialized - for c_type, names in type_info: + for c_type, names, c_tc in type_info: class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter): _immutable_ = True + typecode = c_tc def __init__(self, space, default): self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default)) class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter): @@ -734,40 +737,35 @@ _converters["const "+name+"&"] = ConstRefConverter type_info = ( - (rffi.LONG, ("long", "long int")), - (rffi.LONGLONG, ("long long", "long long int", "Long64_t")), + (rffi.LONG, ("long", "long int"), 'l'), + (rffi.LONGLONG, ("long long", "long long int", "Long64_t"), 'k'), ) - for c_type, names in type_info: + for c_type, names, c_tc in type_info: class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter): _immutable_ = True + typecode = c_tc def __init__(self, space, default): self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default)) - class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter): _immutable_ = True libffitype = jit_libffi.types.pointer - typecode = 'r' - def convert_argument(self, space, w_obj, address, call_local): - x = rffi.cast(self.c_ptrtype, address) - x[0] = self._unwrap_object(space, w_obj) - ba = rffi.cast(rffi.CCHARP, address) - ba[capi.c_function_arg_typeoffset(space)] = self.typecode for name in names: _converters[name] = BasicConverter _converters["const "+name+"&"] = ConstRefConverter # unsigned integer types (use strtoull in setting of default in __init__) type_info = ( - (rffi.USHORT, ("unsigned short", "unsigned short int")), - (rffi.UINT, ("unsigned", "unsigned int")), - (rffi.ULONG, ("unsigned long", "unsigned long int")), - (rffi.ULONGLONG, ("unsigned long long", "unsigned long long int", "ULong64_t")), + (rffi.USHORT, ("unsigned short", "unsigned short int"), 'H'), + (rffi.UINT, ("unsigned", "unsigned int"), 'I'), + (rffi.ULONG, ("unsigned long", "unsigned long int"), 'L'), + (rffi.ULONGLONG, ("unsigned long long", "unsigned long long int", "ULong64_t"), 'K'), ) - for c_type, names in type_info: + for c_type, names, c_tc in type_info: class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter): _immutable_ = True + typecode = c_tc def __init__(self, space, default): self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default)) class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter): @@ -776,6 +774,7 @@ for name in names: _converters[name] = BasicConverter _converters["const "+name+"&"] = ConstRefConverter + _build_basic_converters() # create the array and pointer converters; all real work is in the mixins @@ -789,6 +788,8 @@ ('I', rffi.sizeof(rffi.UINT), ("unsigned int", "unsigned")), ('l', rffi.sizeof(rffi.LONG), ("long int", "long")), ('L', rffi.sizeof(rffi.ULONG), ("unsigned long int", "unsigned long")), + ('k', rffi.sizeof(rffi.LONGLONG),("long long", "long long int", "Long64_t")), + ('K', rffi.sizeof(rffi.ULONGLONG),("unsigned long long", "unsigned long long int", "ULong64_t")), ('f', rffi.sizeof(rffi.FLOAT), ("float",)), ('d', rffi.sizeof(rffi.DOUBLE), ("double",)), ) diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py --- a/pypy/module/cppyy/executor.py +++ b/pypy/module/cppyy/executor.py @@ -318,8 +318,8 @@ (rffi.UINT, capi.c_call_l, ("unsigned", "unsigned int")), (rffi.LONG, capi.c_call_l, ("long", "long int")), (rffi.ULONG, capi.c_call_l, ("unsigned long", "unsigned long int")), - (rffi.LONGLONG, capi.c_call_ll, ("long long", "long long int")), - (rffi.ULONGLONG, capi.c_call_ll, ("unsigned long long", "unsigned long long int")), + (rffi.LONGLONG, capi.c_call_ll, ("long long", "long long int", "Long64_t")), + (rffi.ULONGLONG, capi.c_call_ll, ("unsigned long long", "unsigned long long int", "ULong64_t")), (rffi.FLOAT, capi.c_call_f, ("float",)), (rffi.DOUBLE, capi.c_call_d, ("double",)), ) diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -341,9 +341,33 @@ std::vector& args = *(std::vector*)args_; for ( std::vector::size_type i = 0; i < args.size(); ++i ) { switch ( args[i].fTypeCode ) { + case 'b': /* bool */ + vargs[i] = (void*)&args[i].fValue.fBool; + break; + case 'h': /* short */ + vargs[i] = (void*)&args[i].fValue.fShort; + break; + case 'H': /* unsigned short */ + vargs[i] = (void*)&args[i].fValue.fUShort; + break; + case 'i': /* int */ + vargs[i] = (void*)&args[i].fValue.fInt; + break; + case 'I': /* unsigned int */ + vargs[i] = (void*)&args[i].fValue.fUInt; + break; case 'l': /* long */ vargs[i] = (void*)&args[i].fValue.fLong; break; + case 'L': /* unsigned long */ + vargs[i] = (void*)&args[i].fValue.fULong; + break; + case 'k': /* long long */ + vargs[i] = (void*)&args[i].fValue.fLongLong; + break; + case 'K': /* unsigned long long */ + vargs[i] = (void*)&args[i].fValue.fULongLong; + break; case 'f': /* double */ vargs[i] = (void*)&args[i].fValue.fFloat; break; @@ -353,9 +377,7 @@ case 'D': /* long double */ vargs[i] = (void*)&args[i].fValue.fLongDouble; break; - case 'k': /* long long */ - case 'K': /* unsigned long long */ - case 'U': /* unsigned long */ + case 'o': case 'p': /* void* */ vargs[i] = (void*)&args[i].fValue.fVoidp; break; @@ -519,6 +541,8 @@ // scope reflection information ---------------------------------------------- Bool_t Cppyy::IsNamespace( TCppScope_t scope ) { // Test if this scope represents a namespace. + if ( scope == GLOBAL_HANDLE ) + return kTRUE; TClassRef& cr = type_from_handle( scope ); if ( cr.GetClass() ) return cr->Property() & kIsNamespace; @@ -665,7 +689,7 @@ return (TCppIndex_t)0; } -Cppyy::TCppIndex_t Cppyy::GetMethodIndexAt( TCppScope_t scope, TCppIndex_t imeth) +Cppyy::TCppIndex_t Cppyy::GetMethodIndexAt( TCppScope_t scope, TCppIndex_t imeth ) { TClassRef& cr = type_from_handle (scope); if (cr.GetClass()) @@ -865,21 +889,8 @@ TClassRef& cr = type_from_handle( scope ); if ( cr.GetClass() && cr->GetListOfDataMembers() ) return cr->GetListOfDataMembers()->GetSize(); - else if ( scope == (TCppScope_t)GLOBAL_HANDLE ) { - std::cerr << " global data should be retrieved lazily " << std::endl; - TCollection* vars = gROOT->GetListOfGlobals( kTRUE ); - if ( g_globalvars.size() != (GlobalVars_t::size_type)vars->GetSize() ) { - g_globalvars.clear(); - g_globalvars.reserve(vars->GetSize()); - TIter ivar(vars); - - TGlobal* var = 0; - while ( (var = (TGlobal*)ivar.Next()) ) - g_globalvars.push_back( var ); - } - return (TCppIndex_t)g_globalvars.size(); - } +// global vars (and unknown classes) are always resolved lazily, so report as '0' return (TCppIndex_t)0; } From pypy.commits at gmail.com Thu Jun 30 18:55:21 2016 From: pypy.commits at gmail.com (wlav) Date: Thu, 30 Jun 2016 15:55:21 -0700 (PDT) Subject: [pypy-commit] pypy cling-support: spell out all classes as [A-Z] pattern appears not to be working anymore Message-ID: <5775a359.4346c20a.2650c.ffffc7ff@mx.google.com> Author: Wim Lavrijsen Branch: cling-support Changeset: r85476:7fcccdb6a0b1 Date: 2016-06-30 15:52 -0700 http://bitbucket.org/pypy/pypy/changeset/7fcccdb6a0b1/ Log: spell out all classes as [A-Z] pattern appears not to be working anymore diff --git a/pypy/module/cppyy/test/fragile.xml b/pypy/module/cppyy/test/fragile.xml --- a/pypy/module/cppyy/test/fragile.xml +++ b/pypy/module/cppyy/test/fragile.xml @@ -5,7 +5,22 @@ - + + + + + + + + + + + + + + + +